diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..d6c4d779 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +# Preserve the pinned upstream RoboTwin source byte-for-byte. Its historical +# formatting contains trailing whitespace that must not fail this repository's +# own diff checks. +operator-images/builtin/holobrain-vendor/RoboTwin/** -whitespace diff --git a/backend/operator-runner/test_main.py b/backend/operator-runner/test_main.py index a0c1a543..cf6f4f97 100644 --- a/backend/operator-runner/test_main.py +++ b/backend/operator-runner/test_main.py @@ -3394,6 +3394,26 @@ class ShellCommandOperatorTests(unittest.TestCase): model_dir = root / "model" model_dir.mkdir() (model_dir / "robotwin2_0.config.json").write_text("{}", encoding="utf-8") + robotwin = root / "RoboTwin" + for relative in [ + "task_config/demo_clean.yml", + "task_config/_embodiment_config.yml", + "task_config/_camera_config.yml", + "envs/beat_block_hammer.py", + "description/task_instruction/beat_block_hammer.json", + ]: + path = robotwin / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("{}\n", encoding="utf-8") + (robotwin / "assets" / "embodiments" / "aloha-agilex").mkdir(parents=True) + for relative in [ + "assets/objects/020_hammer/model_data0.json", + "assets/objects/020_hammer/collision/base0.glb", + "assets/objects/020_hammer/visual/base0.glb", + ]: + path = robotwin / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"fixture") export_manifest = { "name": "holobrain-export-manifest.json", "type": "json", @@ -3408,6 +3428,7 @@ class ShellCommandOperatorTests(unittest.TestCase): "label": "HoloBrain RoboTwin 评测", "hyperparams": [ {"key": "ROBOORCHARDLAB_ROOT", "value": str(repo)}, + {"key": "ROBOTWIN_ROOT", "value": str(robotwin)}, {"key": "ENV_JSON", "value": "{\"HOLOBRAIN_OPERATOR_DRY_RUN\":\"1\"}"}, ], }, @@ -3424,6 +3445,7 @@ class ShellCommandOperatorTests(unittest.TestCase): "label": "HoloBrain 模型测试", "hyperparams": [ {"key": "ROBOORCHARDLAB_ROOT", "value": str(repo)}, + {"key": "ROBOTWIN_ROOT", "value": str(robotwin)}, {"key": "ENV_JSON", "value": "{\"HOLOBRAIN_OPERATOR_DRY_RUN\":\"1\"}"}, ], }, diff --git a/backend/operators/builtin/holobrain_operator.py b/backend/operators/builtin/holobrain_operator.py index b7d58d1a..f6f31bf2 100644 --- a/backend/operators/builtin/holobrain_operator.py +++ b/backend/operators/builtin/holobrain_operator.py @@ -17,6 +17,8 @@ from typing import Any, Callable HelperMap = dict[str, Callable[..., Any]] ROBOORCHARDLAB_REPO = "https://github.com/HorizonRobotics/RoboOrchardLab.git" +ROBOTWIN_REPO = "https://github.com/RoboTwin-Platform/RoboTwin.git" +ROBOTWIN_SOURCE_COMMIT = "bf44be51cf5717a5595ce59447f2cf5263d2aa95" ROBOTWIN_DATASET_REPO = "TianxingChen/RoboTwin2.0" HOLOBRAIN_MODEL_REPO = "HorizonRobotics/HoloBrain_v0.0_Qwen" DEFAULT_CONFIG = "projects/holobrain/configs/config_holobrain_qwen_common.py" @@ -30,6 +32,10 @@ DEFAULT_EXPECTED_LMDB_DIRS = ( "aloha_agilex_demo_randomized", "ur5_wsg_demo_clean", ) +TRAINING_DATASET_LMDB_DIRS = { + "robotwin2_0": ("aloha_agilex_demo_clean", "aloha_agilex_demo_randomized"), + "robotwin2_0_ur5_wsg": ("ur5_wsg_demo_clean",), +} DEFAULT_RAW_ROBOTWIN_ZIP_PATTERNS = { "aloha_agilex_demo_clean": ["*aloha-agilex_clean*.zip", "*aloha_agilex_clean*.zip"], "aloha_agilex_demo_randomized": ["*aloha-agilex_randomized*.zip", "*aloha_agilex_randomized*.zip"], @@ -948,6 +954,17 @@ def _latest_checkpoint(workspace: pathlib.Path) -> pathlib.Path | None: return max(candidates, key=_checkpoint_mtime) if candidates else None +def _parent_resume_workspaces() -> list[pathlib.Path]: + parent_output = os.environ.get("PARENT_NODE_OUTPUT_DIR", "").strip() + if parent_output: + return [pathlib.Path(parent_output) / "holobrain-workspace"] + parent_run_root = os.environ.get("PARENT_RUN_WORKSPACE_ROOT", "").strip() + node_id = os.environ.get("NODE_ID", "").strip() + if parent_run_root and node_id: + return [pathlib.Path(parent_run_root) / "nodes" / node_id / "outputs" / "holobrain-workspace"] + return [] + + def _resolve_resume_checkpoint(params: dict[str, str], kwargs: dict[str, Any], workspace: pathlib.Path) -> tuple[pathlib.Path | None, str]: policy = _text(params, "RESUME_POLICY", "auto").lower() if policy not in {"auto", "explicit", "never"}: @@ -966,10 +983,12 @@ def _resolve_resume_checkpoint(params: dict[str, str], kwargs: dict[str, Any], w return selected, "explicit" if policy == "explicit": raise RuntimeError("RESUME_FROM is required when RESUME_POLICY=explicit") - selected = _latest_valid_resume_checkpoint(workspace) - if selected is not None: - kwargs["resume_from"] = str(selected) - return selected, policy + for candidate_workspace in [workspace, *_parent_resume_workspaces()]: + selected = _latest_valid_resume_checkpoint(candidate_workspace) + if selected is not None: + kwargs["resume_from"] = str(selected) + return selected, policy + return None, policy def _select_output_checkpoint(params: dict[str, str], workspace: pathlib.Path) -> tuple[pathlib.Path | None, str]: @@ -1029,7 +1048,35 @@ def _dtype_preflight( return payload, path -def _apply_smoke_training_defaults(params: dict[str, str], kwargs: dict[str, Any]) -> dict[str, Any]: +def _smoke_training_datasets( + params: dict[str, str], + payload: dict[str, Any], + kwargs: dict[str, Any], +) -> list[str]: + configured = kwargs.get("training_datasets") + if configured is not None: + if not isinstance(configured, list) or not all(isinstance(item, str) and item.strip() for item in configured): + raise RuntimeError("CONFIG_KWARGS_JSON.training_datasets must be a non-empty string array") + selected = [item.strip() for item in configured] + if not selected: + raise RuntimeError("CONFIG_KWARGS_JSON.training_datasets must not be empty") + return selected + + expected = _json_list(params, "EXPECTED_LMDB_DIRS_JSON", payload.get("expectedLmdbDirs") or []) + expected_names = {str(item).strip() for item in expected if str(item).strip()} + selected: list[str] = [] + for dataset_name in DEFAULT_EXPECTED_DATASETS: + required = set(TRAINING_DATASET_LMDB_DIRS[dataset_name]) + if required & expected_names: + selected.append(dataset_name) + return selected or ["robotwin2_0"] + + +def _apply_smoke_training_defaults( + params: dict[str, str], + kwargs: dict[str, Any], + training_datasets: list[str] | None = None, +) -> dict[str, Any]: next_kwargs = dict(kwargs) max_step = max(1, _int(params, "SMOKE_MAX_STEP", 2)) next_kwargs.setdefault("max_step", max_step) @@ -1037,8 +1084,9 @@ def _apply_smoke_training_defaults(params: dict[str, str], kwargs: dict[str, Any next_kwargs.setdefault("step_log_freq", max(1, min(max_step, _int(params, "SMOKE_LOG_STEP_FREQ", 1)))) next_kwargs.setdefault("batch_size", max(1, _int(params, "SMOKE_BATCH_SIZE", 1))) next_kwargs.setdefault("num_workers", max(0, _int(params, "SMOKE_NUM_WORKERS", 0))) - next_kwargs.setdefault("training_datasets", ["robotwin2_0"]) - next_kwargs.setdefault("deploy_datasets", ["robotwin2_0"]) + selected = list(training_datasets or ["robotwin2_0"]) + next_kwargs.setdefault("training_datasets", selected) + next_kwargs.setdefault("deploy_datasets", selected) return next_kwargs @@ -1379,7 +1427,7 @@ def run_data_prepare( effective_params["ARROW_MAX_EPISODES"] = str(max(1, _int(params, "SMOKE_MAX_EPISODES", 1))) output_path = pathlib.Path(output_dir) output_path.mkdir(parents=True, exist_ok=True) - default_expected = [DEFAULT_EXPECTED_LMDB_DIRS[0]] if training_mode == "smoke" else list(DEFAULT_EXPECTED_LMDB_DIRS) + default_expected = ["ur5_wsg_demo_clean"] if training_mode == "smoke" else list(DEFAULT_EXPECTED_LMDB_DIRS) expected = tuple( str(item).strip() for item in _json_list(effective_params, "EXPECTED_LMDB_DIRS_JSON", default_expected) @@ -1490,24 +1538,40 @@ def _materialize_data_for_repo(repo_root: pathlib.Path, params: dict[str, str], raise RuntimeError(f"RoboTwin data root does not exist: {source}") target = repo_root / "projects" / "holobrain" / "data" / "robotwin2.0" if training_mode == "smoke": - # The official robotwin2_0 config names both clean and randomized - # directories. A real smoke subset may intentionally contain only a - # clean LMDB. Build an idempotent overlay and map the second config - # path to the same real subset instead of mutating the source mount. + kwargs = _json_dict(params, "CONFIG_KWARGS_JSON", {}) + selected_datasets = _smoke_training_datasets(params, payload, kwargs) + unsupported = [name for name in selected_datasets if name not in TRAINING_DATASET_LMDB_DIRS] + if unsupported: + raise RuntimeError( + "Smoke training only supports these RoboTwin datasets: " + + ", ".join(DEFAULT_EXPECTED_DATASETS) + + f"; unsupported: {', '.join(unsupported)}" + ) + # Build an idempotent overlay without mutating the source mount. The + # official Aloha config names clean and randomized paths, so a real + # clean-only smoke subset may safely expose the same samples at both + # paths. UR5/WSG uses a single real directory and needs no alias. if target.is_symlink() or (target.exists() and not target.is_dir()): target.unlink() target.mkdir(parents=True, exist_ok=True) for child in sorted(source.iterdir()): if child.is_dir(): _safe_link_or_copy(child, target / child.name) - clean = target / "aloha_agilex_demo_clean" - randomized = target / "aloha_agilex_demo_randomized" - if clean.exists() and not randomized.exists(): - _safe_link_or_copy(clean, randomized) - if not clean.exists() or not randomized.exists(): + if "robotwin2_0" in selected_datasets: + clean = target / "aloha_agilex_demo_clean" + randomized = target / "aloha_agilex_demo_randomized" + if clean.exists() and not randomized.exists(): + _safe_link_or_copy(clean, randomized) + required_dirs = { + directory + for dataset_name in selected_datasets + for directory in TRAINING_DATASET_LMDB_DIRS[dataset_name] + } + missing = sorted(directory for directory in required_dirs if not (target / directory).exists()) + if missing: raise RuntimeError( - "Smoke training requires a real aloha_agilex_demo_clean LMDB; " - "the idempotent RoboTwin config overlay could not be initialized" + "Smoke training data does not satisfy the selected RoboTwin config; " + f"missing LMDB directories: {', '.join(missing)}" ) else: _safe_link_or_copy(source, target) @@ -1538,7 +1602,11 @@ def run_train( logging_dir.mkdir(parents=True, exist_ok=True) kwargs = _json_dict(params, "CONFIG_KWARGS_JSON", {}) if training_mode == "smoke": - kwargs = _apply_smoke_training_defaults(params, kwargs) + kwargs = _apply_smoke_training_defaults( + params, + kwargs, + _smoke_training_datasets(params, data_payload, kwargs), + ) resume_checkpoint, resume_policy = _resolve_resume_checkpoint(params, kwargs, workspace) initial_checkpoint = _text(params, "INITIAL_CHECKPOINT_PATH") if initial_checkpoint: @@ -1684,6 +1752,102 @@ def _model_dir_from_inputs(params: dict[str, str], inputs: list[dict[str, Any]], raise RuntimeError("MODEL_DIR or upstream HoloBrain export manifest is required") +def _select_inference_prefix(params: dict[str, str], export_payload: dict[str, Any], training_mode: str) -> str: + available = [str(item).strip() for item in export_payload.get("inferencePrefixes", []) if str(item).strip()] + configured = _text(params, "INFERENCE_PREFIX") + if configured: + if available and configured not in available: + raise RuntimeError( + f"INFERENCE_PREFIX {configured!r} is not present in the exported model; available: {', '.join(available)}" + ) + return configured + if len(available) == 1: + return available[0] + preferred = "robotwin2_0_ur5_wsg" if training_mode == "smoke" else "robotwin2_0" + if not available or preferred in available: + return preferred + raise RuntimeError("INFERENCE_PREFIX is required when the exported model contains multiple non-default prefixes") + + +def _prepare_robotwin_runtime( + params: dict[str, str], + output_path: pathlib.Path, + task_names: list[str], + inference_prefix: str, +) -> tuple[pathlib.Path, pathlib.Path, dict[str, Any]]: + source_root = _path(_text(params, "ROBOTWIN_ROOT", os.environ.get("RoboTwin_PATH", "/opt/RoboTwin"))) + asset_root_text = _text(params, "ROBOTWIN_ASSET_ROOT") + asset_root = _path(asset_root_text) if asset_root_text else source_root / "assets" + runtime_root = source_root + errors: list[str] = [] + required_source_paths = [ + source_root / "task_config" / "demo_clean.yml", + source_root / "task_config" / "_embodiment_config.yml", + source_root / "task_config" / "_camera_config.yml", + ] + for task_name in task_names: + required_source_paths.extend( + [ + source_root / "envs" / f"{task_name}.py", + source_root / "description" / "task_instruction" / f"{task_name}.json", + ] + ) + missing_source = [str(path) for path in required_source_paths if not path.exists()] + if missing_source: + errors.append("missing RoboTwin source files: " + ", ".join(missing_source)) + + if asset_root_text and asset_root.resolve() != (source_root / "assets").resolve(): + runtime_root = output_path / "robotwin-runtime" + runtime_root.mkdir(parents=True, exist_ok=True) + for child in sorted(source_root.iterdir()) if source_root.is_dir() else []: + if child.name in {".git", "assets"}: + continue + _safe_link_or_copy(child, runtime_root / child.name) + if asset_root.exists(): + _safe_link_or_copy(asset_root, runtime_root / "assets") + + if inference_prefix == "robotwin2_0_ur5_wsg": + embodiment = "ur5-wsg" + errors.append( + "RoboOrchardLab's current RoboTwin evaluator only accepts a combined dual-arm URDF, while official " + "UR5-Wsg uses separate left/right embodiments; training and export are supported, but simulator " + "evaluation remains blocked until split-layout support is implemented and validated" + ) + else: + embodiment = "aloha-agilex" + required_asset_paths = [asset_root / "embodiments" / embodiment] + if "beat_block_hammer" in task_names: + required_asset_paths.extend( + [ + asset_root / "objects" / "020_hammer" / "model_data0.json", + asset_root / "objects" / "020_hammer" / "collision" / "base0.glb", + asset_root / "objects" / "020_hammer" / "visual" / "base0.glb", + ] + ) + missing_assets = [str(path) for path in required_asset_paths if not path.exists()] + if missing_assets: + errors.append("missing RoboTwin simulation assets: " + ", ".join(missing_assets)) + payload = { + "schemaVersion": "droboticflow.holobrain.robotwin-preflight/v1", + "valid": not errors, + "repository": ROBOTWIN_REPO, + "sourceCommit": ROBOTWIN_SOURCE_COMMIT, + "sourceRoot": str(source_root), + "runtimeRoot": str(runtime_root), + "assetRoot": str(asset_root), + "inferencePrefix": inference_prefix, + "embodiment": embodiment, + "taskNames": task_names, + "errors": errors, + "checkedAt": _utc_now(), + } + preflight_path = output_path / "holobrain-robotwin-preflight.json" + _write_json_atomic(preflight_path, payload) + if errors: + raise RuntimeError("HoloBrain RoboTwin preflight failed: " + "; ".join(errors)) + return runtime_root, preflight_path, payload + + def run_export( run_id: str, node_id: str, @@ -1815,12 +1979,19 @@ def run_robotwin_eval( export_payload = _first_payload(inputs, helpers, "holobrain-export-manifest") training_mode = _training_mode(params, str(export_payload.get("trainingMode") or "full")) model_dir = _model_dir_from_inputs(params, inputs, helpers) + inference_prefix = _select_inference_prefix(params, export_payload, training_mode) output_json = output_path / "robotwin-eval-result.json" task_names = ( [_text(params, "SMOKE_TASK_NAME", "beat_block_hammer")] if training_mode == "smoke" else [str(item) for item in _json_list(params, "TASK_NAMES_JSON", []) if str(item).strip()] ) + robotwin_root, robotwin_preflight_path, robotwin_preflight = _prepare_robotwin_runtime( + params, + output_path, + task_names, + inference_prefix, + ) gpu_ids = [str(item) for item in _json_list(params, "GPU_IDS_JSON", [0]) if str(item).strip() != ""] eval_mode = _text(params, "SMOKE_EVAL_MODE", "local") if training_mode == "smoke" else _text(params, "EVAL_MODE", "ray") if eval_mode not in {"local", "ray"}: @@ -1832,7 +2003,7 @@ def run_robotwin_eval( "--model_dir", str(model_dir), "--inference_prefix", - _text(params, "INFERENCE_PREFIX", "robotwin2_0"), + inference_prefix, "--episode_num", str(episode_num), "--mode", @@ -1854,6 +2025,7 @@ def run_robotwin_eval( if _bool(params, "SAVE_VIDEO", False): command.append("--save_video") env = _command_env(params, output_path) + env["RoboTwin_PATH"] = str(robotwin_root) command_path, stdout_path, stderr_path = _run_command( command, project_dir, @@ -1866,6 +2038,8 @@ def run_robotwin_eval( "taskNames": task_names, "episodeNum": episode_num, "evalMode": eval_mode, + "inferencePrefix": inference_prefix, + "robotwinPreflight": robotwin_preflight, }, ) metrics: dict[str, Any] = {} @@ -1882,7 +2056,9 @@ def run_robotwin_eval( "taskNames": task_names, "episodeNum": episode_num, "device": device, + "inferencePrefix": inference_prefix, "syntheticResult": False, + "robotwinPreflight": robotwin_preflight, }, "modelDir": str(model_dir), "metrics": metrics, @@ -1897,7 +2073,13 @@ def run_robotwin_eval( output_path, "holobrain-robotwin-eval-report.json", payload, - [(command_path, "json"), (stdout_path, "text"), (stderr_path, "text"), (output_json, "json")], + [ + (command_path, "json"), + (stdout_path, "text"), + (stderr_path, "text"), + (output_json, "json"), + (robotwin_preflight_path, "json"), + ], node_data, ) @@ -1918,8 +2100,14 @@ def run_inference_test( export_payload = _first_payload(inputs, helpers, "holobrain-export-manifest") training_mode = _training_mode(params, str(export_payload.get("trainingMode") or "full")) model_dir = _model_dir_from_inputs(params, inputs, helpers) - inference_prefix = _text(params, "INFERENCE_PREFIX", "robotwin2_0") + inference_prefix = _select_inference_prefix(params, export_payload, training_mode) task_name = _text(params, "TASK_NAME", "beat_block_hammer") + robotwin_root, robotwin_preflight_path, robotwin_preflight = _prepare_robotwin_runtime( + params, + output_path, + [task_name], + inference_prefix, + ) episode_num = max(1, _int(params, "EPISODE_NUM", 1)) inference_mode = _text(params, "INFERENCE_MODE", "local").lower() if inference_mode not in {"local", "ray"}: @@ -1954,6 +2142,7 @@ def run_inference_test( raise RuntimeError("GPU_IDS_JSON must contain at least one GPU id when INFERENCE_MODE=ray") command.extend(["--gpu_ids", *gpu_ids]) env = _command_env(params, output_path) + env["RoboTwin_PATH"] = str(robotwin_root) command_path, stdout_path, stderr_path = _run_command( command, project_dir, @@ -1966,6 +2155,8 @@ def run_inference_test( "taskName": task_name, "episodeNum": episode_num, "inferenceMode": inference_mode, + "inferencePrefix": inference_prefix, + "robotwinPreflight": robotwin_preflight, }, ) dry_run = str(env.get("HOLOBRAIN_OPERATOR_DRY_RUN", "")).strip().lower() in TRUE_VALUES @@ -1989,6 +2180,7 @@ def run_inference_test( "episodeNum": episode_num, "mode": inference_mode, "device": device, + "robotwinPreflight": robotwin_preflight, }, "commandPath": str(command_path), "stdoutPath": str(stdout_path), @@ -2001,6 +2193,12 @@ def run_inference_test( output_path, "holobrain-inference-result.json", payload, - [(command_path, "json"), (stdout_path, "text"), (stderr_path, "text"), (metrics_path, "json")], + [ + (command_path, "json"), + (stdout_path, "text"), + (stderr_path, "text"), + (metrics_path, "json"), + (robotwin_preflight_path, "json"), + ], node_data, ) diff --git a/internal/api/builtin_operators_catalog.json b/internal/api/builtin_operators_catalog.json index cf914a1f..98a7208e 100644 --- a/internal/api/builtin_operators_catalog.json +++ b/internal/api/builtin_operators_catalog.json @@ -10590,7 +10590,7 @@ "value": "" }, { - "description": "可选覆盖 LMDB 目录;空值时 smoke 只要求 aloha_agilex_demo_clean,full 要求三个官方目录", + "description": "可选覆盖 LMDB 目录;空值时 smoke 默认使用三相机 UR5/WSG 真实小数据,full 要求三个官方目录", "key": "EXPECTED_LMDB_DIRS_JSON", "type": "json", "value": "" @@ -11061,6 +11061,18 @@ "type": "string", "value": "/opt/RoboOrchardLab" }, + { + "description": "镜像内固定版本的 RoboTwin 源码根目录", + "key": "ROBOTWIN_ROOT", + "type": "string", + "value": "/opt/RoboTwin" + }, + { + "description": "可选只读仿真资产根目录;为空时使用 ROBOTWIN_ROOT/assets", + "key": "ROBOTWIN_ASSET_ROOT", + "type": "string", + "value": "" + }, { "description": "导出模型目录;为空时消费上游导出 manifest", "key": "MODEL_DIR", @@ -11068,10 +11080,10 @@ "value": "" }, { - "description": "导出 pipeline config prefix", + "description": "导出 pipeline config prefix;空值时从上游 export manifest 自动选择", "key": "INFERENCE_PREFIX", "type": "string", - "value": "robotwin2_0" + "value": "" }, { "description": "official eval mode:ray 或 local", @@ -11242,6 +11254,18 @@ "type": "string", "value": "/opt/RoboOrchardLab" }, + { + "description": "镜像内固定版本的 RoboTwin 源码根目录", + "key": "ROBOTWIN_ROOT", + "type": "string", + "value": "/opt/RoboTwin" + }, + { + "description": "可选只读仿真资产根目录;为空时使用 ROBOTWIN_ROOT/assets", + "key": "ROBOTWIN_ASSET_ROOT", + "type": "string", + "value": "" + }, { "description": "导出模型目录;为空时消费上游导出 manifest", "key": "MODEL_DIR", @@ -11249,10 +11273,10 @@ "value": "" }, { - "description": "导出 pipeline config prefix", + "description": "导出 pipeline config prefix;空值时从上游 export manifest 自动选择", "key": "INFERENCE_PREFIX", "type": "string", - "value": "robotwin2_0" + "value": "" }, { "description": "实际执行的 RoboTwin 推理任务", diff --git a/internal/worker/executor.go b/internal/worker/executor.go index 1052b110..89b25b86 100644 --- a/internal/worker/executor.go +++ b/internal/worker/executor.go @@ -44,6 +44,9 @@ type stageManifestRecord struct { var protectedSystemEnv = map[string]struct{}{ "WORKFLOW_PREVIOUS_RUN_TIMESTAMP": {}, "WORKFLOW_CURRENT_RUN_TIMESTAMP": {}, + "PARENT_RUN_ID": {}, + "PARENT_RUN_WORKSPACE_ROOT": {}, + "PARENT_NODE_OUTPUT_DIR": {}, "OPERATOR_ALLOW_SHELL_COMMANDS": {}, "OPERATOR_COMMAND_ALLOWLIST_REGEX": {}, "OPERATOR_COMMAND_DENYLIST_REGEX": {}, @@ -392,6 +395,12 @@ func buildNodeEnvWithResources(run *domain.RunRecord, workflow *domain.WorkflowR "OUTPUT_DIR": workspace.OutputDir, "WORKSPACE_ROOT": workspace.RunRoot, } + if parentRunID := strings.TrimSpace(run.ParentRunID); parentRunID != "" { + parentRunRoot := filepath.Join(filepath.Dir(workspace.RunRoot), filepath.Base(parentRunID)) + env["PARENT_RUN_ID"] = parentRunID + env["PARENT_RUN_WORKSPACE_ROOT"] = parentRunRoot + env["PARENT_NODE_OUTPUT_DIR"] = filepath.Join(parentRunRoot, "nodes", node.ID, "outputs") + } if run.Trigger == domain.RunTriggerScheduled { env["WORKFLOW_CURRENT_RUN_TIMESTAMP"] = runTimestampEnvValue(firstNonEmpty(run.CurrentRunAt, run.CreatedAt)) env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"] = runTimestampEnvValue(run.PreviousRunAt) diff --git a/internal/worker/executor_test.go b/internal/worker/executor_test.go index f84b7327..722ab87d 100644 --- a/internal/worker/executor_test.go +++ b/internal/worker/executor_test.go @@ -15,6 +15,7 @@ func TestBuildNodeEnvInjectsPlatformMongoEnvForMongoNode(t *testing.T) { run := &domain.RunRecord{ ID: "run-1", + ParentRunID: "run-parent", WorkflowID: "workflow-1", Trigger: domain.RunTriggerScheduled, PreviousRunAt: "2026-04-21T10:00:00Z", @@ -31,7 +32,10 @@ func TestBuildNodeEnvInjectsPlatformMongoEnvForMongoNode(t *testing.T) { Label: "MongoDB 写入", Category: "DATA", Description: "write docs", - Hyperparams: []domain.HyperParam{{Key: "MONGO_COLLECTION", Value: "qualityReports"}}, + Hyperparams: []domain.HyperParam{ + {Key: "MONGO_COLLECTION", Value: "qualityReports"}, + {Key: "PARENT_RUN_ID", Value: "user-value-must-not-win"}, + }, }, } @@ -78,6 +82,15 @@ func TestBuildNodeEnvInjectsPlatformMongoEnvForMongoNode(t *testing.T) { if env["WORKSPACE_ROOT"] != workspace.RunRoot { t.Fatalf("expected WORKSPACE_ROOT to point at run root, got %q", env["WORKSPACE_ROOT"]) } + if env["PARENT_RUN_ID"] != "run-parent" { + t.Fatalf("expected protected parent run id, got %q", env["PARENT_RUN_ID"]) + } + if env["PARENT_RUN_WORKSPACE_ROOT"] != "/tmp/workspace/system/runs/run-parent" { + t.Fatalf("expected parent run workspace root, got %q", env["PARENT_RUN_WORKSPACE_ROOT"]) + } + if env["PARENT_NODE_OUTPUT_DIR"] != "/tmp/workspace/system/runs/run-parent/nodes/node-1/outputs" { + t.Fatalf("expected parent node output directory, got %q", env["PARENT_NODE_OUTPUT_DIR"]) + } expectedPrevious := "2026-04-21T18:00:00+08:00" if env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"] != expectedPrevious { t.Fatalf("expected previous run timestamp env, got %q", env["WORKFLOW_PREVIOUS_RUN_TIMESTAMP"]) diff --git a/operator-images/builtin/holobrain-base.Dockerfile b/operator-images/builtin/holobrain-base.Dockerfile index 0143f1c1..655a5444 100644 --- a/operator-images/builtin/holobrain-base.Dockerfile +++ b/operator-images/builtin/holobrain-base.Dockerfile @@ -3,11 +3,15 @@ FROM pytorch/pytorch:2.6.0-cuda12.4-cudnn9-runtime WORKDIR /app ENV PYTHONUNBUFFERED=1 \ - PYTHONPATH=/app:/opt/RoboOrchardLab \ + PYTHONPATH=/app:/opt/RoboOrchardLab:/opt/RoboTwin \ + PIP_DEFAULT_TIMEOUT=120 \ + PIP_RETRIES=10 \ USER=droboticflow \ LOGNAME=droboticflow \ HOME=/tmp/droboticflow-home \ ROBOORCHARDLAB_ROOT=/opt/RoboOrchardLab \ + RoboTwin_PATH=/opt/RoboTwin \ + NVIDIA_DRIVER_CAPABILITIES=compute,utility,graphics \ HF_HOME=/opt/hf-cache \ HF_HUB_CACHE=/opt/hf-cache/hub \ TRANSFORMERS_CACHE=/opt/hf-cache/transformers \ @@ -19,7 +23,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ git \ libglib2.0-0 \ libgl1 \ + libvulkan1 \ linux-libc-dev \ + mesa-vulkan-drivers \ python3-dev \ && rm -rf /var/lib/apt/lists/* RUN printf 'torch==2.6.0\ntorchvision==0.21.0\n' > /tmp/holobrain-constraints.txt @@ -61,6 +67,7 @@ RUN python -m pip install --no-cache-dir -c /tmp/holobrain-constraints.txt \ "websockets>=10.0" COPY operator-images/builtin/holobrain-vendor/robo_orchard_core /opt/robo_orchard_core COPY operator-images/builtin/holobrain-vendor/RoboOrchardLab /opt/RoboOrchardLab +COPY operator-images/builtin/holobrain-vendor/RoboTwin /opt/RoboTwin RUN cd /opt/robo_orchard_core && \ python -m pip install --no-cache-dir -e . --no-deps && \ cd /opt/RoboOrchardLab && \ @@ -73,4 +80,4 @@ ENV HF_HUB_OFFLINE=1 \ HF_DATASETS_OFFLINE=1 RUN mkdir -p /opt/RoboOrchardLab/projects/holobrain/data /tmp/droboticflow-home /workspace /outputs && \ chown -R 10001:10001 /opt/RoboOrchardLab/projects/holobrain/data /tmp/droboticflow-home && \ - chmod -R a+rX /opt/RoboOrchardLab /opt/hf-cache /opt/models + chmod -R a+rX /opt/RoboOrchardLab /opt/RoboTwin /opt/hf-cache /opt/models diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/.gitignore b/operator-images/builtin/holobrain-vendor/RoboTwin/.gitignore new file mode 100644 index 00000000..8958179d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/.gitignore @@ -0,0 +1,30 @@ +models/ +data/* +!data/process_stuck.py +__pycache__/ +**/checkpoints/ +result/ +envs/curobo +*.zip +viewer_show.* +eval_video/ +task_config/ +**/.DS_Store + +# eval result +eval_result/ + +# Code Generation +assets/* +!assets/_download.py +!assets/files + +policy/weights/* + +envs/curobo/* + +.vscode +/config.json + +/*.json* +/*.txt \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/DROBOTICFLOW_SOURCE.json b/operator-images/builtin/holobrain-vendor/RoboTwin/DROBOTICFLOW_SOURCE.json new file mode 100644 index 00000000..9498374b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/DROBOTICFLOW_SOURCE.json @@ -0,0 +1,8 @@ +{ + "repository": "https://github.com/RoboTwin-Platform/RoboTwin.git", + "release": "release", + "commit": "bf44be51cf5717a5595ce59447f2cf5263d2aa95", + "importedFor": "HoloBrain RoboTwin evaluation runtime", + "assetPolicy": "Simulation assets are not vendored; mount or provision them separately under assets/.", + "licenseFile": "LICENSE" +} diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/LICENSE b/operator-images/builtin/holobrain-vendor/RoboTwin/LICENSE new file mode 100644 index 00000000..5f0a83b7 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Tianxing Chen (陈天行) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/README.md b/operator-images/builtin/holobrain-vendor/RoboTwin/README.md new file mode 100644 index 00000000..97611681 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/README.md @@ -0,0 +1,175 @@ +

+ RoboTwin Bimanual Robotic Manipulation Platform
+

+

Lastest Version: RoboTwin 2.0
🤲 Webpage | Document | Paper | Community | Leaderboard

+ +https://private-user-images.githubusercontent.com/88101805/463126988-e3ba1575-4411-4a36-ad65-f0b2f49890c3.mp4 + +**[2.0 Version (lastest)]** RoboTwin 2.0: A Scalable Data Generator and Benchmark with Strong Domain Randomization for Robust Bimanual Robotic Manipulation
+Under Review 2025: [Webpage](https://robotwin-platform.github.io/) | [Document](https://robotwin-platform.github.io/doc) | [PDF](https://arxiv.org/pdf/2506.18088) | [arXiv](https://arxiv.org/abs/2506.18088) | [Talk (in Chinese)](https://www.bilibili.com/video/BV18p3izYE63/?spm_id_from=333.337.search-card.all.click) | [机器之心](https://mp.weixin.qq.com/s/SwORezmol2Qd9YdrGYchEA) | [Leaderboard](https://robotwin-platform.github.io/leaderboard)
+> Tianxing Chen\*, Zanxin Chen\*, Baijun Chen\*, Zijian Cai\*, Yibin Liu\*, Qiwei Liang, Zixuan Li, Xianliang Lin, Yiheng Ge, Zhenyu Gu, Weiliang Deng, Yubin Guo, Tian Nian, Xuanbing Xie, Qiangyu Chen, Kailun Su, Tianling Xu, Guodong Liu, Mengkang Hu, Huan-ang Gao, Kaixuan Wang, Zhixuan Liang, Yusen Qin, Xiaokang Yang, Ping Luo†, Yao Mu† + +**[RoboTwin Dual-Arm Collaboration Challenge@CVPR'25 MEIS Workshop]** RoboTwin Dual-Arm Collaboration Challenge Technical Report at CVPR 2025 MEIS Workshop
+Official Technical Report: [PDF](https://arxiv.org/pdf/2506.23351) | [arXiv](https://arxiv.org/abs/2506.23351) | [量子位](https://mp.weixin.qq.com/s/qxqs9vvvHsAJ-0hoYANYzQ)
+ +**[1.0 Version]** RoboTwin: Dual-Arm Robot Benchmark with Generative Digital Twins
+Accepted to CVPR 2025 (Highlight): [PDF](https://arxiv.org/pdf/2504.13059) | [arXiv](https://arxiv.org/abs/2504.13059)
+> Yao Mu* †, Tianxing Chen* , Zanxin Chen* , Shijia Peng* , Zhiqian Lan, Zeyu Gao, Zhixuan Liang, Qiaojun Yu, Yude Zou, Mingkun Xu, Lunkai Lin, Zhiqiang Xie, Mingyu Ding, Ping Luo†. + +**[Early Version]** RoboTwin: Dual-Arm Robot Benchmark with Generative Digital Twins (early version)
+Accepted to ECCV Workshop 2024 (Best Paper Award): [PDF](https://arxiv.org/pdf/2409.02920) | [arXiv](https://arxiv.org/abs/2409.02920)
+> Yao Mu* †, Tianxing Chen* , Shijia Peng*, Zanxin Chen*, Zeyu Gao, Zhiqian Lan, Yude Zou, Lunkai Lin, Zhiqiang Xie, Ping Luo†. + + + +# 📚 Overview + +| Branch Name | Link | +|-------------|------| +| 2.0 Version Branch | [main](https://github.com/RoboTwin-Platform/RoboTwin/tree/main) (latest) | +| IsaacLab-Arena Branch | [IsaacLab-Arena](https://github.com/RoboTwin-Platform/RoboTwin/tree/IsaacLab-Arena) | +| RLinf Branch | [RLinf_support](https://github.com/RoboTwin-Platform/RoboTwin/tree/RLinf_support) | +| WBCD 2026 Branch | [WBCD-2026](https://github.com/RoboTwin-Platform/RoboTwin/tree/WBCD-2026) | +| 1.0 Version Branch | [1.0 Version](https://github.com/RoboTwin-Platform/RoboTwin/tree/RoboTwin-1.0) | +| 1.0 Version Code Generation Branch | [1.0 Version GPT](https://github.com/RoboTwin-Platform/RoboTwin/tree/gpt) | +| Early Version Branch | [Early Version](https://github.com/RoboTwin-Platform/RoboTwin/tree/early_version) | +| 第十九届“挑战杯”人工智能专项赛分支 | [Challenge-Cup-2025](https://github.com/RoboTwin-Platform/RoboTwin/tree/Challenge-Cup-2025) | +| CVPR 2025 Challenge Round 1 Branch | [CVPR-Challenge-2025-Round1](https://github.com/RoboTwin-Platform/RoboTwin/tree/CVPR-Challenge-2025-Round1) | +| CVPR 2025 Challenge Round 2 Branch | [CVPR-Challenge-2025-Round2](https://github.com/RoboTwin-Platform/RoboTwin/tree/CVPR-Challenge-2025-Round2) | + + + +# 🐣 Update +* **2026/02/20**, Usage supported in StarVLA, which is a user-friendly codebase for VLA development. +* **2026/01/23**, We update IsaacLab-Arena and RLinf support (contributed by RLinf team). +* **2025/08/28**, We update the RoboTwin 2.0 Paper [PDF](https://arxiv.org/pdf/2506.18088). +* **2025/08/25**, We fix ACT deployment code and update the [leaderboard](https://robotwin-platform.github.io/leaderboard). +* **2025/08/06**, We release RoboTwin 2.0 Leaderboard: [leaderboard website](https://robotwin-platform.github.io/leaderboard). +* **2025/07/23**, RoboTwin 2.0 received Outstanding Poster at ChinaSI 2025 (Ranking 1st). +* **2025/07/19**, We Fix DP3 evaluation code error. We will update RoboTwin 2.0 paper next week. +* **2025/07/09**, We update endpose control mode, please see [[RoboTwin Doc - Usage - Control Robot](https://robotwin-platform.github.io/doc/usage/control-robot.html)] for more details. +* **2025/07/08**, We upload [Challenge-Cup-2025](https://github.com/RoboTwin-Platform/RoboTwin/tree/Challenge-Cup-2025) Branch (第十九届挑战杯分支). +* **2025/07/02**, Fix Piper Wrist Bug [[issue](https://github.com/RoboTwin-Platform/RoboTwin/issues/104)]. Please redownload the embodiment asset. +* **2025/07/01**, We release Technical Report of RoboTwin Dual-Arm Collaboration Challenge @ CVPR 2025 MEIS Workshop [[arXiv](https://arxiv.org/abs/2506.23351)] ! +* **2025/06/21**, We release RoboTwin 2.0 [[Webpage](https://robotwin-platform.github.io/)] ! +* **2025/04/11**, RoboTwin is seclected as CVPR Highlight paper! +* **2025/02/27**, RoboTwin is accepted to CVPR 2025 ! +* **2024/09/30**, RoboTwin (Early Version) received the Best Paper Award at the ECCV Workshop! +* **2024/09/20**, Officially released RoboTwin. + +# 🛠️ Installation + +See [RoboTwin 2.0 Document (Usage - Install & Download)](https://robotwin-platform.github.io/doc/usage/robotwin-install.html) for installation instructions. It takes about 20 minutes for installation. + +# 🤷‍♂️ Tasks Informations +See [RoboTwin 2.0 Tasks Doc](https://robotwin-platform.github.io/doc/tasks/index.html) for more details. + +

+ +

+ +# 🧑🏻‍💻 Usage + +## Document + +> Please Refer to [RoboTwin 2.0 Document (Usage)](https://robotwin-platform.github.io/doc/usage/index.html) for more details. + +## Data Collection +We provide over 100,000 pre-collected trajectories as part of the open-source release [RoboTwin Dataset](https://huggingface.co/datasets/TianxingChen/RoboTwin2.0/tree/main/dataset). +However, we strongly recommend users to perform data collection themselves due to the high configurability and diversity of task and embodiment setups. + +description + +## 1. Task Running and Data Collection +Running the following command will first search for a random seed for the target collection quantity, and then replay the seed to collect data. + +``` +bash collect_data.sh ${task_name} ${task_config} ${gpu_id} +# Example: bash collect_data.sh beat_block_hammer demo_randomized 0 +``` + +## 2. Modify Task Config +☝️ See [RoboTwin 2.0 Tasks Configurations Doc](https://robotwin-platform.github.io/doc/usage/configurations.html) for more details. + +# 🚴‍♂️ Policy Baselines +## Policies Support +[DP](https://robotwin-platform.github.io/doc/usage/DP.html), [ACT](https://robotwin-platform.github.io/doc/usage/ACT.html), [DP3](https://robotwin-platform.github.io/doc/usage/DP3.html), [RDT](https://robotwin-platform.github.io/doc/usage/RDT.html), [PI0](https://robotwin-platform.github.io/doc/usage/Pi0.html), [OpenVLA-oft](https://robotwin-platform.github.io/doc/usage/OpenVLA-oft.html) + +[TinyVLA](https://robotwin-platform.github.io/doc/usage/TinyVLA.html), [DexVLA](https://robotwin-platform.github.io/doc/usage/DexVLA.html) (Contributed by Media Group) + +[LLaVA-VLA](https://robotwin-platform.github.io/doc/usage/LLaVA-VLA.html) (Contributed by IRPN Lab, HKUST(GZ)) + +[GO-1](https://robotwin-platform.github.io/doc/usage/GO1.html) (Contributed by GO-1 Team) + +Deploy Your Policy: [Guidance](https://robotwin-platform.github.io/doc/usage/deploy-your-policy.html) + +⏰ TODO: G3Flow, HybridVLA, SmolVLA, AVR, UniVLA + +# 🏄‍♂️ Experiment & LeaderBoard + +> We recommend that the RoboTwin Platform can be used to explore the following topics: +> 1. single - task fine - tuning capability +> 2. visual robustness +> 3. language diversity robustness (language condition) +> 4. multi-tasks capability +> 5. cross-embodiment performance + +The full leaderboard and setting can be found in: [https://robotwin-platform.github.io/leaderboard](https://robotwin-platform.github.io/leaderboard). + +# 💽 Pre-collected Large-scale Dataset + +Please refer to [RoboTwin 2.0 Dataset - Huggingface](https://huggingface.co/datasets/TianxingChen/RoboTwin2.0/tree/main/dataset). + +# 👍 Citations +If you find our work useful, please consider citing: + +RoboTwin 2.0: A Scalable Data Generator and Benchmark with Strong Domain Randomization for Robust Bimanual Robotic Manipulation +``` +@article{chen2025robotwin, + title={Robotwin 2.0: A scalable data generator and benchmark with strong domain randomization for robust bimanual robotic manipulation}, + author={Chen, Tianxing and Chen, Zanxin and Chen, Baijun and Cai, Zijian and Liu, Yibin and Li, Zixuan and Liang, Qiwei and Lin, Xianliang and Ge, Yiheng and Gu, Zhenyu and others}, + journal={arXiv preprint arXiv:2506.18088}, + year={2025} +} +``` + +RoboTwin: Dual-Arm Robot Benchmark with Generative Digital Twins, accepted to CVPR 2025 (Highlight) +``` +@InProceedings{Mu_2025_CVPR, + author = {Mu, Yao and Chen, Tianxing and Chen, Zanxin and Peng, Shijia and Lan, Zhiqian and Gao, Zeyu and Liang, Zhixuan and Yu, Qiaojun and Zou, Yude and Xu, Mingkun and Lin, Lunkai and Xie, Zhiqiang and Ding, Mingyu and Luo, Ping}, + title = {RoboTwin: Dual-Arm Robot Benchmark with Generative Digital Twins}, + booktitle = {Proceedings of the Computer Vision and Pattern Recognition Conference (CVPR)}, + month = {June}, + year = {2025}, + pages = {27649-27660} +} +``` + +Benchmarking Generalizable Bimanual Manipulation: RoboTwin Dual-Arm Collaboration Challenge at CVPR 2025 MEIS Workshop +``` +@article{chen2025benchmarking, + title={Benchmarking Generalizable Bimanual Manipulation: RoboTwin Dual-Arm Collaboration Challenge at CVPR 2025 MEIS Workshop}, + author={Chen, Tianxing and Wang, Kaixuan and Yang, Zhaohui and Zhang, Yuhao and Chen, Zanxin and Chen, Baijun and Dong, Wanxi and Liu, Ziyuan and Chen, Dong and Yang, Tianshuo and others}, + journal={arXiv preprint arXiv:2506.23351}, + year={2025} +} +``` + +RoboTwin: Dual-Arm Robot Benchmark with Generative Digital Twins (early version), accepted to ECCV Workshop 2024 (Best Paper Award) +``` +@article{mu2024robotwin, + title={RoboTwin: Dual-Arm Robot Benchmark with Generative Digital Twins (early version)}, + author={Mu, Yao and Chen, Tianxing and Peng, Shijia and Chen, Zanxin and Gao, Zeyu and Zou, Yude and Lin, Lunkai and Xie, Zhiqiang and Luo, Ping}, + journal={arXiv preprint arXiv:2409.02920}, + year={2024} +} +``` + +# 😺 Acknowledgement + +**Software Support**: D-Robotics, **Hardware Support**: AgileX Robotics, **AIGC Support**: Deemos. + +Contact [Tianxing Chen](https://tianxingchen.github.io) if you have any questions or suggestions. + +# 🏷️ License +This repository is released under the MIT license. See [LICENSE](./LICENSE) for additional details. diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/assets/_download.py b/operator-images/builtin/holobrain-vendor/RoboTwin/assets/_download.py new file mode 100644 index 00000000..203d166f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/assets/_download.py @@ -0,0 +1,9 @@ +from huggingface_hub import snapshot_download + +snapshot_download( + repo_id="TianxingChen/RoboTwin2.0", + allow_patterns=["background_texture.zip", "embodiments.zip", "objects.zip"], + local_dir=".", + repo_type="dataset", + resume_download=True, +) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/assets/files/50_tasks.gif b/operator-images/builtin/holobrain-vendor/RoboTwin/assets/files/50_tasks.gif new file mode 100644 index 00000000..d5b0f24c Binary files /dev/null and b/operator-images/builtin/holobrain-vendor/RoboTwin/assets/files/50_tasks.gif differ diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/assets/files/domain_randomization.png b/operator-images/builtin/holobrain-vendor/RoboTwin/assets/files/domain_randomization.png new file mode 100644 index 00000000..37de4bae Binary files /dev/null and b/operator-images/builtin/holobrain-vendor/RoboTwin/assets/files/domain_randomization.png differ diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/__init__.py b/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/__init__.py new file mode 100644 index 00000000..9b46a880 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/__init__.py @@ -0,0 +1,10 @@ +# Description: This file is used to import all the necessary files for the gpt_api module. +from .gpt_agent import * # Core GPT agent logic +from .prompt import * # Prompt templates and formatting utilities +from .task_info import * # Task metadata, descriptions, and configurations + +# Try importing optional observation handling module +try: + from .observation_agent import * # Optional: multimodal or perception-specific agent interface +except ImportError as e: + print(f"Warning: Failed to import observation_agent module: {e}") \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/gpt_agent.py b/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/gpt_agent.py new file mode 100644 index 00000000..3a42e75f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/gpt_agent.py @@ -0,0 +1,37 @@ +from openai import OpenAI + +kimi_api = "Your key" +openai_api = "Your key" +deep_seek_api = "Your key" + +# Configure the API and key (using DeepSeek as an example) +def generate(message, gpt="deepseek", temperature=0): + + if gpt == "deepseek": + MODEL = "deepseek-chat" + OPENAI_API_BASE = "https://api.deepseek.com" + # Set your API key here + OPENAI_API_KEY = deep_seek_api + client = OpenAI(api_key=OPENAI_API_KEY, base_url=OPENAI_API_BASE) + + elif gpt == "openai": + MODEL = "gpt-4o" + OPENAI_API_BASE = "https://api.gptapi.us/v1" + OPENAI_API_KEY = openai_api + client = OpenAI(api_key=OPENAI_API_KEY, base_url=OPENAI_API_BASE) + + else: + raise ValueError(f"Unsupported API provider: {gpt}") + + print('start generating') + response = client.chat.completions.create( + model=MODEL, + messages=message, + stream=False, + temperature=temperature, + ) + print('end generating') + + return response.choices[0].message.content + + diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/observation_agent.py b/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/observation_agent.py new file mode 100644 index 00000000..f987f3bc --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/observation_agent.py @@ -0,0 +1,240 @@ +import base64 +import os +import glob +from openai import OpenAI + +from gpt_agent import kimi_api, openai_api, deep_seek_api, generate + + +def observe_task_execution(episode_id, task_name, task_info, problematic_code=None, save_dir="./camera_images", camera_name=None, generate_dir_name=None): + """ + Observe task execution by analyzing step-by-step images using an image understanding API. + + Args: + episode_id (int): ID of the episode to analyze. + task_name (str): Name of the task. + task_info (dict): Basic information about the task. + problematic_code (str, optional): Potentially faulty code generated in a previous step. + save_dir (str): Base directory where images are saved. + camera_name (str): Name of the camera used to capture the images. + generate_dir_name (str, optional): Name of the subdirectory with generated images. + + Returns: + str: Textual description of the observation result. + """ + client = OpenAI( + api_key=kimi_api, + base_url="https://api.moonshot.cn/v1", + ) + + # Check if the save_dir already contains the task name + base_task_name = task_name.lower() if task_name else "" + if base_task_name and os.path.basename(save_dir) == base_task_name: + # If task name is already included in save_dir, use it directly + task_dir = save_dir + else: + # Otherwise, append task name to the path + task_dir = os.path.join(save_dir, base_task_name) if base_task_name else save_dir + + # If a generated subdirectory name is specified, add it to the path + if generate_dir_name: + task_dir = os.path.join(task_dir, generate_dir_name) + + print(f"Looking for task images in: {os.path.abspath(task_dir)}") + + # Check if task directory exists + if not os.path.exists(task_dir): + return f"Error: Image directory not found at {task_dir}" + + # Get images for the specific episode + image_files = sorted(glob.glob(os.path.join(task_dir, f"episode{episode_id}_*.png"))) + + if not image_files: + return f"Error: No images found for episode {episode_id} in directory {task_dir}" + + # Extract step names from image filenames + step_names = [] + for f in image_files: + filename = os.path.basename(f) + first_underscore_pos = filename.find('_') + if first_underscore_pos != -1: + step_name = filename[first_underscore_pos+1:].rsplit('.', 1)[0] + step_names.append(step_name) + else: + step_names.append(filename.rsplit('.', 1)[0]) + + # Logging for debugging purposes (from observation_agent.py) + print(f"Image search pattern: episode{episode_id}_*.png, number of files found: {len(image_files)}") + # for f in image_files[:5]: # Uncomment to print first 5 filenames + # print(f" - {os.path.basename(f)}") + + # Construct the prompt + prompt = f"""Analyze the execution of the following robot task: +Task name: {task_name} +Task description: {task_info.get('description', 'No description provided')} +Task goal: {task_info.get('goal', 'No goal provided')} + +You will be shown images from each step of the task execution. Please analyze: +1. Whether each step was executed successfully. +2. If any step failed, identify which one and explain why. +3. Whether the overall task was successfully completed. +4. If the task failed, provide detailed reasoning. + +You will see execution images for the following steps: {', '.join(step_names)} +""" + + if problematic_code: + prompt += f"\nHere is a piece of potentially problematic code:\n```python\n{problematic_code}\n```\nPlease analyze if the code is related to the observed issue." + + # Prepare message content for API call + user_content = [] + + # Add textual prompt + user_content.append({ + "type": "text", + "text": prompt + }) + + # Add images and step names + for img_path in image_files: + filename = os.path.basename(img_path) + first_underscore_pos = filename.find('_') + if first_underscore_pos != -1: + step_name = filename[first_underscore_pos+1:].rsplit('.', 1)[0] + else: + step_name = filename.rsplit('.', 1)[0] + + # Add step name + user_content.append({ + "type": "text", + "text": f"Step: {step_name}" + }) + + # Add image as base64 + try: + base64_image = encode_image(img_path) + user_content.append({ + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{base64_image}" + } + }) + except Exception as e: + print(f"Warning: Failed to encode image {img_path}: {str(e)}") + + # Call the image analysis API + try: + response = client.chat.completions.create( + model="moonshot-v1-32k-vision-preview", + messages=[ + {"role": "system", "content": "You are a robot task execution analysis expert. Please analyze the provided image sequence."}, + {"role": "user", "content": user_content} + ] + ) + return response.choices[0].message.content + except Exception as e: + error_msg = f"Error occurred while calling the image understanding API: {str(e)}" + print(error_msg) + return error_msg + + +def encode_image(image_path): + """Encode an image file to a base64 string.""" + with open(image_path, "rb") as image_file: + return base64.b64encode(image_file.read()).decode("utf-8") + + +def insert_observation_points(task_info, task_code, generate_num_id=0): + """ + Insert observation function calls at key points in robot task code. + + Args: + task_info (dict): Information about the task + task_code (str): Original code for the task + + Returns: + str: Code with inserted observation points and steps summary + """ + + # Extract task name + if isinstance(task_info, dict) and 'task_name' in task_info: + task_name = task_info.get('task_name') + else: + # Try to extract from code + import re + task_name_match = re.search(r'class\s+gpt_(\w+)', task_code) + task_name = task_name_match.group(1) if task_name_match else "unknown_task" + + # Prepare the prompt for the LLM + prompt = f"""You are an expert in robot programming. I have a robot task code that needs observation functions added for monitoring. + +Task information: +{task_info} + +I need you to: +1. Identify ONLY the main logical steps in this task implementation that cause SIGNIFICANT SCENE CHANGES +2. After each such logical step in the code, insert a camera observation function with this format: + `self.save_camera_images(task_name="{task_name}", step_name="stepX_descriptive_name", generate_num_id="generate_num_{generate_num_id}")` + where X is the sequential step number and descriptive_name is a brief description of what just happened +3. Provide a numbered list of all the steps you've identified in the task +4. ADD AN OBSERVATION AT THE BEGINNING OF THE TASK to capture the initial scene state +5. ADD AN OBSERVATION AT THE END OF THE TASK to capture the final scene state + +Here's the current code: +```python +{task_code} +``` + +IMPORTANT CONSTRAINTS: +- ADD FEWER THAN 10 OBSERVATION POINTS in total +- ONLY add observations after operations that cause VISIBLE SCENE CHANGES +- Do NOT add observations for planning, calculations, or any operations that don't visibly change the scene +- Focus on key state changes like: robot arm movements, gripper operations, object manipulations +- Skip observations for intermediate movements, planning steps, or calculations +- The observation function is already defined in the code +- Give each step a descriptive name like "gripper_closed", "move_to_target", etc. +- The step number (X in stepX) should increase sequentially +- DO NOT MODIFY ANY EXISTING ROBOT OPERATION CODE - only insert observation function calls after existing code without changing the original functionality + +Format your response as follows: + +STEP_LIST: +1. First step description +2. Second step description +... + +MODIFIED_CODE: +```python + +``` +""" + + # Get the modified code from LLM in one call + response = generate(message=[{ + "role": "system", + "content": "You are an AI assistant that helps with programming robot tasks." + }, { + "role": "user", + "content": prompt + }]) + + # Extract the step list and modified code + try: + steps_part, code_part = response.split("MODIFIED_CODE:", 1) + steps = steps_part.replace("STEP_LIST:", "").strip() + modified_code = code_part.strip() + + # Clean up any potential markdown code block formatting + if modified_code.startswith("```python"): + modified_code = modified_code[len("```python"):].strip() + if modified_code.endswith("```"): + modified_code = modified_code[:-3].strip() + except ValueError: + # Fallback in case the format isn't as expected + steps = "Failed to extract step list" + modified_code = response + + # Format the output + output = f"# task_name: {task_name}\n# task_step:\n{steps}\n\n# task_code:\n```python\n{modified_code}\n```" + + return output diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/prompt.py b/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/prompt.py new file mode 100644 index 00000000..150f2d35 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/prompt.py @@ -0,0 +1,345 @@ +# ====================== PROMPT ============================= + +BASIC_INFO = ''' +In this environment, distance 1 indicates 1 meter long. Pose is representated as 7 dimention, [x, y, z, qw, qx, qy, qz]. +For a 7-dimensional Pose object, you can use Pose.p to get the [x, y, z] coordinates and Pose.q to get the [qw, qx, qy, qz] quaternion orientation. +All functions which has parameter actor, and all of actor should be in the Actor object. +In the world coordinate system, the positive directions of the xyz coordinate axes are right, front, and upper respectively, so the direction vectors on the right, front, +and upper sides are [1,0,0], [0,1,0], [0,0,1] respectively. In the same way, we can get the unit vectors of the left side, back side and down side. +Each actor in the environment has one or more functional points, which are specific locations designed for interactions. +Access functional points using actor.get_functional_point(point_id, return_type), where return_type can be "pose", "p", or "q". +''' + +CODE_TEMPLATE = ''' +from envs._base_task import Base_Task +from envs.$TASK_NAME$ import $TASK_NAME$ +from envs.utils import * +import sapien + +class gpt_$TASK_NAME$($TASK_NAME$): + def play_once(self): + pass +''' + +AVAILABLE_ENV_FUNCTION = { + "open_gripper": + "def open_gripper(self, arm_tag: ArmTag, pos=1.) -> tuple[ArmTag, list[Action]].\ + Opens the gripper of the specified arm.\ + Returns: tuple[ArmTag, list[Action]] containing the gripper-open action.\ + Args:\ + arm_tag: Which arm's gripper to open\ + pos: Gripper position (1 = fully open)", + "close_gripper": + "def close_gripper(self, arm_tag: ArmTag, pos=0.) -> tuple[ArmTag, list[Action]].\ + Closes the gripper of the specified arm.\ + Returns: tuple[ArmTag, list[Action]] containing the gripper-close action.\ + Args:\ + arm_tag: Which arm's gripper to close\ + pos: Gripper position (0 = fully closed)", + "move": + "def move(self, actions_by_arm1: tuple[ArmTag, list[Action]], actions_by_arm2: tuple[ArmTag, list[Action]] = None).\ + Executes action sequences on one or both robotic arms simultaneously.\ + No Return.\ + Args:\ + actions_by_arm1: Action sequence for the first arm, formatted as (arm_tag, [action1, action2, ...])\ + actions_by_arm2: Optional, action sequence for the second arm", + + # "move_to_pose": + # "def move_to_pose(self, arm_tag: ArmTag, target_pose: list) -> tuple[ArmTag, list[Action]].\ + # Moves the end-effector of the specified arm to a specific absolute pose.\ + # Returns: tuple[ArmTag, list[Action]] containing the move-to-pose actions.\ + # Args:\ + # arm_tag: The arm to control\ + # target_pose: Absolute position and/or orientation, length 3 or 7 (xyz + optional quaternion)", + + # "move_by_displacement": + # "def move_by_displacement(self, arm_tag: ArmTag, x=0., y=0., z=0., quat=None, move_axis='world') -> tuple[ArmTag, list[Action]].\ + # Moves the end-effector of the specified arm along relative directions and sets its orientation.\ + # Returns: tuple[ArmTag, list[Action]] containing the move-by-displacement actions.\ + # Args:\ + # arm_tag: The arm to control\ + # x, y, z: Displacement along each axis (in meters)\ + # quat: Optional quaternion specifying the target orientation; if not set, uses current orientation\ + # move_axis: 'world' means displacement is in world coordinates, 'arm' means displacement is in local coordinates",\ + "move_by_displacement": + "def move_by_displacement(self, arm_tag: ArmTag, z=0., move_axis='world') -> tuple[ArmTag, list[Action]].\ + Moves the end-effector of the specified arm along relative directions and sets its orientation.\ + Returns: tuple[ArmTag, list[Action]] containing the move-by-displacement actions.\ + Args:\ + arm_tag: The arm to control\ + z: Displacement along the z-axis (in meters)\ + move_axis: 'world' means displacement is in world coordinates, 'arm' means displacement is in local coordinates", + "grasp_actor": + "def grasp_actor(self, actor: Actor, arm_tag: ArmTag, pre_grasp_dis=0.1, grasp_dis=0, gripper_pos=0., contact_point_id=None) -> tuple[ArmTag, list[Action]].\ + Generates a sequence of actions to pick up the specified Actor.\ + Returns: tuple[ArmTag, list[Action]] containing the grasp actions.\ + Args:\ + actor: The object to grasp\ + arm_tag: Which arm to use\ + pre_grasp_dis: Pre-grasp distance (default 0.1 meters), the arm will move to this position first\ + grasp_dis: Grasping distance (default 0 meters), the arm moves from the pre-grasp position to this position and then closes the gripper\ + gripper_pos: Gripper closing position (default 0, fully closed)\ + contact_point_id: Optional list of contact point IDs; if not provided, the best grasping point is selected automatically", + "place_actor": + "def place_actor(self, actor: Actor, arm_tag: ArmTag, target_pose: list | np.ndarray, functional_point_id: int = None, pre_dis=0.1, dis=0.02, is_open=True, **kwargs) -> tuple[ArmTag, list[Action]].\ + Places a currently held object at a specified target pose.\ + Returns: tuple[ArmTag, list[Action]] containing the place actions.\ + Args: \ + actor: The currently held object\ + arm_tag: The arm holding the object\ + target_pose: Target position/orientation, It is recommended to use the return value of actor.get_functional_point(..., 'pose') or pose in actor_list as target_pose\ + functional_point_id: Optional ID of the functional point; if provided, aligns this point to the target, otherwise aligns the base of the object\ + pre_dis: Pre-place distance (default 0.1 meters), arm moves to this position first\ + dis: Final placement distance (default 0.02 meters), arm moves from pre-place to this location, then opens the gripper\ + is_open: Whether to open the gripper after placing (default True), Set False if you need to keep gripper closed to maintain hold of the object\ + **kwargs: Other optional parameters:\ + constrain : {'free', 'align', 'auto'}, default='auto' Alignment strategy:\ + 'free': Only forces the object's z-axis to align with the target point's z-axis, other axes are determined by projection.\ + 'align': Forces all axes of the object to align with all axes of the target point.\ + 'auto': Automatically selects a suitable placement pose based on grasp direction (vertical or horizontal).\ + pre_dis_axis : {'grasp', 'fp'} or np.ndarray or list, default='grasp'. Specifies the pre-placement offset direction.", + "back_to_origin": + "def back_to_origin(self, arm_tag: ArmTag) -> tuple[ArmTag, list[Action]].\ + Returns the specified arm to its predefined initial position.\ + Returns: tuple[ArmTag, list[Action]] containing the return-to-origin action.\ + Args:\ + arm_tag: The arm to return to origin", + + # "get_arm_pose": + # "def get_arm_pose(self, arm_tag: ArmTag) -> list[float].\ + # Gets the current pose of the end-effector of the specified arm.\ + # Returns: A list of 7 floats: [x, y, z, qw, qx, qy, qz], representing position and orientation.\ + # Args:\ + # arm_tag: Which arm to query", +} + +FUNCTION_EXAMPLE = ''' +You can directly use the actors provided in the actor_list: +```python +# For example, if actor_list contains ["self.object1", "self.object2"] +# You can directly use: +object1 = self.hammer +object2 = self.block +``` + +# Using ArmTag class to represent arms: +arm_tag = ArmTag("left") # Left arm +arm_tag = ArmTag("right") # Right arm + +# Example of selecting an arm based on conditions: +arm_tag = ArmTag("left" if actor_position[0] < 0 else "right") + +# Each actor in the environment may have multiple functional points that are useful for different interactions. +# Functional points provide precise locations for interactions like grasping, placing, or aligning objects. + +# To get a functional point from an actor: +```python +functional_point_pose = actor.get_functional_point(point_id, "pose") # Returns a complete 7-dimensional Pose object with p (position) and q (orientation) +position = functional_point_pose.p # Get [x, y, z] position of the functional point +orientation = functional_point_pose.q # Get [qw, qx, qy, qz] quaternion orientation of the functional point +``` +Note: The pose from a functional point is already set according to the expected alignment/direction for the task. For placement, use get_functional_point(point_id, "pose") directly—do NOT construct or rotate your own quaternion. + +# When stacking one object on top of another (for example, placing blockA on top of blockB): +target_pose = self.last_actor.get_functional_point(point_id, "pose") +# Use this target_pose in place_actor to place the object exactly on top of last_actor at the specified functional point. +```python +self.move( + self.place_actor( + actor=self.current_actor, # The object to be placed + target_pose=target_pose, # The pose acquired from last_actor + arm_tag=arm_tag, + functional_point_id=0, # Align functional point 0, or specify as needed + pre_dis=0.1, + dis=0.02, + pre_dis_axis="fp", # Use functional point direction for pre-displacement, if the functional point is used + ) +) +``` + +For all actors in `actor_list` that are of type `pose`, such as `middle_pose` or `actor_target_pose`, these are already `Pose` objects (or lists of `Pose`), so you do **not** need to call `.get_pose()` again. You can pass them directly as `target_pose`. +Example: +```python +# Place the actor at actor_pose (already a Pose object) +self.move( + self.place_actor( + self.box, + target_pose=self.actor_pose, # already a Pose, no need for get_pose() + arm_tag=grasp_arm_tag, + functional_point_id=0, # functional_point_id can be retrived from the actor list if the actor has functional points + pre_dis=0, + dis=0, # set dis to 0 if is_open is False, and the gripper will not open after placing. Set the `dis` to a small value like 0.02 if you want the gripper to open after placing. + is_open=False, # if is_open is False, pre_dis and dis will be 0, and the gripper will not open after placing. + constrain="free", # if task requires the object to be placed in a specific pose that mentioned in the task description (like "the head of the actor should be toward xxx), you can set constrain to "align", in all of other cases, you should set constrain to "free". + pre_dis_axis='fp', # Use functional point direction for pre-displacement, if the functional_point_id is used + ) +) +``` +Note: For the `target_actor`, It's a actor not a Pose, so you need to call `get_pose()` to get its pose. or call `get_functional_point()` to get its functional point. + + +For the grasping of a certain actor, you can check its position to decide which arm to use: +```python +# Get the actor's pose +actor_pose = self.actor.get_pose() # Use actor_pose.p for position, actor_pose.q for orientation +actor_position = actor_pose.p # [x, y, z] + +# Example of selecting an arm based on conditions: +arm_tag = ArmTag("left" if actor_position[0] < 0 else "right") + +# Grasp actor with selected arm +self.move( + self.grasp_actor(actor=self.actor, arm_tag=arm_tag) +) +``` + +Here are some APIs and examples of grasping objects: +If you want to grasp an actor, you typically execute the following code: +```python +# Or grasp with arm_tag +self.move( + self.grasp_actor( + actor=self.actor, + arm_tag=arm_tag, # arm_tag can be ArmTag("left") or ArmTag("right") + pre_grasp_dis=0.1, + grasp_dis=0 + ) +) +``` + +If you want to pick up an actor and lift it, you can refer to the following sample code: +```python +# Grasp the object +self.move( + self.grasp_actor( + actor=self.actor, + arm_tag=arm_tag, # arm_tag can be ArmTag("left") or ArmTag("right") + pre_grasp_dis=0.1, + grasp_dis=0 + ) +) + +# Lift the object up by moving relative to current position, you should lift the arm up evrery time after grasping an object to avoid collision. +self.move( + self.move_by_displacement( + arm_tag=arm_tag, + z=0.07, # Move 7cm upward + move_axis='world' + ) +) +``` +The code for grasping with the right arm is similar to the above code. + +Here are some examples of gripper control: +```python +# Open gripper fully +self.move( + self.open_gripper(arm_tag=arm_tag, pos=1.0) # arm_tag can be ArmTag("left") or ArmTag("right") +) + +# Open gripper halfway +self.move( + self.open_gripper(arm_tag=arm_tag, pos=0.5) # arm_tag can be ArmTag("left") or ArmTag("right") +) + +# Close gripper fully +self.move( + self.close_gripper(arm_tag=arm_tag, pos=0.0) # arm_tag can be ArmTag("left") or ArmTag("right") +) + +# Close gripper halfway +self.move( + self.close_gripper(arm_tag=arm_tag, pos=0.5) # arm_tag can be ArmTag("left") or ArmTag("right") +) +``` + +Here are some APIs and examples of placing objects: +To place an object at a target location, you typically execute the following code: +```python +# Place the object at a specific target pose +self.move( + self.place_actor( + actor=self.actor, + arm_tag=arm_tag, + target_pose=self.target_pose, # self.target_pose can be retrived from the actor list. + functional_point_id=0, # functional_point_id can be retrived from the actor list if the actor has functional points + pre_dis=0.1, + dis=0.02, # set dis to 0 if is_open is False, and the gripper will not open after placing. Set the `dis` to a small value like 0.02 if you want the gripper to open after placing. + is_open=True, # Controls gripper state after placing: True to release object (default), False to maintain grip on object + pre_dis_axis='fp', # Use functional point direction for pre-displacement, if the functional_point_id is used + ) +) + +# Lift the gripper up after placing to avoid collision with the object. (Only needed if is_open is True when placing, which means the object is released) +self.move( + self.move_by_displacement( + arm_tag=arm_tag, + z=0.07, # Move 7cm upward + move_axis='world' # Move in world coordinates + ) +``` + +If you want to align a functional point of the object with the target, you can specify the functional_point_id: +```python +# Place the object by aligning functional point 0 with the target pose +self.move( + self.place_actor( + actor=self.actor, + arm_tag=arm_tag, + target_pose=target_pose, + functional_point_id=0, # functional_point_id can be retrived from the actor list if the actor has functional points + pre_dis=0.1, + dis=0.02, # set dis to 0 if is_open is False, and the gripper will not open after placing. + pre_dis_axis='fp' # Use functional point direction for pre-displacement, if the functional_point_id is used + ) +) +``` + +If both arms need to work together simultaneously, use the move() function with two arm actions: +```python +# Move both arms simultaneously +left_arm_tag = ArmTag("left") +right_arm_tag = ArmTag("right") +self.move( + self.grasp_actor(actor=self.left_actor, arm_tag=left_arm_tag), + self.grasp_actor(actor=self.right_actor, arm_tag=right_arm_tag) +) + +# Lift both actors up after grasping +self.move( + self.move_by_displacement(arm_tag=left_arm_tag, z=0.07), # Move left arm up by 10cm + self.move_by_displacement(arm_tag=right_arm_tag, z=0.07) # Move right arm up by 10cm +) +``` + + +Place left object while moving right arm back to origin +```python +move_arm_tag = ArmTag("left") # Specify which arm is placing the object +back_arm_tag = ArmTag("right") # Specify which arm is moving back to origin +self.move( + self.place_actor( + actor=self.left_actor, + arm_tag=move_arm_tag, + target_pose=target_pose, + pre_dis_axis="fp", + ), + self.back_to_origin(arm_tag=back_arm_tag) +) +``` +The code for placing with the right arm is similar to the above code. + +To return arms to their initial positions: +```python +# Return arm to origin +self.move(self.back_to_origin(arm_tag=arm_tag)) + +# Return both arms to origin simultaneously +left_arm_tag = ArmTag("left") +right_arm_tag = ArmTag("right") +self.move( + self.back_to_origin(arm_tag=left_arm_tag), + self.back_to_origin(arm_tag=right_arm_tag) +) +``` +''' diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/run_code.py b/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/run_code.py new file mode 100644 index 00000000..7bb63565 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/run_code.py @@ -0,0 +1,114 @@ +import os +import yaml +import sys +import importlib +import argparse + +from gpt_agent import * +from prompt import * +from task_info import * +from test_gen_code import setup_task_config, run + +# Global variable definitions +SCRIPT_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "script") +CONFIGS_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "task_config") + + +def run_code(task_info, las_error=None, message=None): + """ + Execute generated code for a robot task based on task information and previous errors. + + Args: + task_info (dict): Dictionary containing task metadata (name, description, etc.). + las_error (str, optional): Last recorded error message, if any. + message (list, optional): Message history for the agent. + + Returns: + tuple: (success_rate, error_message, error_count, run_records) + """ + check_num = 50 + if message is None: + message = [] + + print("Running code for task:", task_info['task_name']) + + # Extract task info + task_name = task_info['task_name'] + task_description = task_info['task_description'] + + print("Task Name:", task_name) + print("Task Description:", task_description) + + task, args = setup_task_config(task_name) + + try: + # Updated to match the new return values of run() + success_rate, error_message, error_count, run_records = run(task, args, check_num) + return success_rate, error_message, error_count, run_records + + except KeyboardInterrupt: + print("Testing interrupted by user.") + return 0, "Testing interrupted by user", 20 + + except Exception as e: + import traceback + error_trace = traceback.format_exc() + print(f"An error occurred during testing: {e}\n{error_trace}") + return 0, f"Error during testing: {e}", 20 + + +def main(task_info_dic): + """ + Main function to test generated code for a given robot task. + + Args: + task_info_dic (dict): Dictionary containing task information. + """ + task_info = now_task_info = task_info_dic + messages = [{ + "role": "system", + "content": "You need to generate relevant code for some robot tasks in a robot simulation environment based on the provided API." + }] + las_error_message = None + + # Run the test + success_rate, las_error_message, error_count, run_records = run_code( + now_task_info, las_error_message, messages + ) + + # Evaluate result + if success_rate >= 0.5: + print(f"Successfully generated and executed code for task: {task_info['task_name']}") + else: + print(f"Failed to generate or execute code for task: {task_info['task_name']}") + print("Error message:\n", las_error_message) + now_task_info["task_description"] = ( + f"Failed to generate code, error message: {las_error_message}, " + f"error count: {str(error_count)}" + ) + now_task_info["current_code"] = None + + print("Final Success Rate:", success_rate) + + +if __name__ == "__main__": + # Parse command-line arguments + parser = argparse.ArgumentParser(description='Run generated code for a robot task.') + parser.add_argument('task_name', type=str) + now_task = None + + # Get task info from task name string + try: + task_name = parser.parse_args().task_name.upper() + exec(f'now_task = {task_name}') + except Exception as e: + raise ValueError("Invalid task name specified.") from e + + # Run main function + main(now_task) + + +""" +Usage: +python code_gen/run_code.py task_name +""" \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/task_generation.py b/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/task_generation.py new file mode 100644 index 00000000..5080fb59 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/task_generation.py @@ -0,0 +1,236 @@ +import sys +import os +import json + +# Add the project root directory to the system path +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from gpt_agent import * +from prompt import * +from task_info import * +from test_gen_code import * + +import argparse + +def generate_code(task_info, las_error=None, message=None): + """Generate code for robot task based on task info and previous errors.""" + if message is None: + message = [] + + # Extract task information + task_name = task_info['task_name'] + task_description = task_info['task_description'] + current_code = task_info['current_code'] + + # Get the enriched actor_list + original_actor_list = task_info['actor_list'] + actor_list = enrich_actors(original_actor_list) + + # print("actor_list: ", actor_list) + + available_env_function = str(AVAILABLE_ENV_FUNCTION) + function_example = str(FUNCTION_EXAMPLE) + + # Generate code based on error status + if las_error is not None: + # Handle error case - provide error info to improve generation + Prompt = ( + f"The code is unsuccessful, \n# Last Error Message: \n{las_error}\n\n" + f"# Task description: \n{task_description}\n\n" + f"# Actor List: \n{actor_list}\n\n" + ) + else: + # First attempt case - create initial code file + res = f''' +from envs._base_task import Base_Task +from envs.{task_name} import {task_name} +from envs.utils import * +import sapien + +class gpt_{task_name}({task_name}): + def play_once(self): + pass + ''' + file_name = f"envs_gen/gpt_{task_name}.py" + with open(file_name, 'w') as file: + file.write(res) + + # Construct full prompt with all necessary information + Prompt = ( + f"{BASIC_INFO}\n\n" + f"# Task description: \n{task_description}\n\n" + f"# Actor List: \n{actor_list}\n\n" + f"# Available API: \n{available_env_function}\n\n" + f"# Function Example: \n{function_example}\n\n" + f"# Current Code:\n{current_code}" + ) + + # Add prompt to message history + message.append({"role": "user", "content": Prompt}) + + # Generate code using the model + res = generate(message, gpt="deepseek", temperature=0) + + # Extract the relevant portion of the generated code + res = f''' +from envs._base_task import Base_Task +from envs.{task_name} import {task_name} +from envs.utils import * +import sapien + +class gpt_{task_name}({task_name}): + ''' + res[res.find('def play_once'):res.rfind("```")] + + # Save generated code to file + file_name = f"envs_gen/gpt_{task_name}.py" + with open(file_name, 'w') as file: + file.write(res) + + print("Task Name: ", task_name) + print("Task Description: ", task_description) + + task, args = setup_task_config(task_name) + + try: + # Update this section to match the new return values of the run function + success_rate, error_message, error_count, run_records = run(task, args) + + return res, success_rate, error_message, error_count, run_records + except KeyboardInterrupt: + print("Test interrupted by user") + return res, 0, "Test interrupted by user", 20 + except Exception as e: + import traceback + error_trace = traceback.format_exc() + print(f"Error occurred during testing: {e}\n{error_trace}") + return res, 0, f"Error occurred during testing: {e}", 20 + + +def main(task_info_dic): + """Main function to generate and test code for robot tasks.""" + # Initialize variables + task_info = now_task_info = task_info_dic + messages = [{"role": "system", "content": "You need to generate relevant code for some robot tasks in a robot simulation environment based on the provided API."}] + generate_num = 5 + success_threshold = 0.5 + las_error_message = None + suc_list = [] + task_name = task_info['task_name'] + task_description = task_info['task_description'] + + # Store the best code and its success rate + best_code = None + best_success_rate = 0 + best_run_records = None + + # Create log file name with timestamp + import datetime + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + log_dir = "envs_gen/logs" + os.makedirs(log_dir, exist_ok=True) + log_filename = f"{log_dir}/{task_info['task_name']}_{timestamp}.log" + + # Store all attempt records + all_attempts = [] + + # Try multiple generations until success or limit reached + for id in range(generate_num): + print(f"Generate code for task: {task_info['task_name']} ({id+1}/{generate_num})") + + # Generate and test code + res_code, success_rate, las_error_message, error_count, run_records = generate_code( + now_task_info, las_error_message, messages + ) + + # Track success rates + suc_list.append(success_rate) + + # Record this attempt + attempt_record = { + "attempt_id": id + 1, + "success_rate": success_rate, + "error_message": las_error_message, + "error_count": error_count, + "code": res_code, + "run_records": run_records + } + all_attempts.append(attempt_record) + + # Save best code + if success_rate > best_success_rate: + best_success_rate = success_rate + best_code = res_code + best_run_records = run_records + print(f"New best code found, success rate: {best_success_rate}") + + # Check if generation was successful + if success_rate >= success_threshold: + print(f"Successfully generated code for task: {task_info['task_name']}") + break + + # Handle failure case + print(f"Failed to generate code for task: {task_name} (attempt {id+1})\nError message: \n{las_error_message}") + + # Update task description and code for the next attempt + print(f"Failed to generate code for task: {task_info['task_name']} {id}\nError massage: \n{las_error_message}") + change_info = """The error may be caused by: +1. pre_dis_axis is not set correctly in the place_actor function; +2. the functional point is not set correctly in the place_actor function; +3. The pre_dis or dis is not set correctly in the place_actor function; +4. The constrain is not set correctly in the place_actor function, free or align is not constantly fixed, if the code did not have above error, please try to set the constrain to another value. +5. The code didn't take into account the note given in the example function. +The task can be accomplished only through the existing API and example function, please do not use any other API that is not listed in the available API list and examples.\n""" + now_task_info["task_description"] = f"{task_description}\nFailed to generate code, error message: {las_error_message}, error count: {str(error_count)}\n" + change_info + now_task_info["current_code"] = res_code + + # Ensure the final saved code is the best one + if best_code is not None: + task_name = task_info['task_name'] + file_name = f"envs_gen/gpt_{task_name}.py" + print(f"Saving best code, success rate: {best_success_rate}") + with open(file_name, 'w') as file: + file.write(best_code) + + print(f"Best success rate: {best_success_rate}") + print(f"All success rates: {suc_list}") + + # Save log data to file + with open(log_filename, 'w') as log_file: + log_data = { + "task_name": task_info['task_name'], + "task_description": task_info['task_description'], + "best_success_rate": best_success_rate, + "success_rates": suc_list, + "best_code": best_code, + "best_run_records": best_run_records, + "all_attempts": all_attempts + } + json.dump(log_data, log_file, indent=2) + + print(f"Log has been saved to: {log_filename}") + + return best_success_rate, suc_list, best_code, best_run_records + + +if __name__ == "__main__": + # Parse command line arguments + parser = argparse.ArgumentParser(description='Process some integers.') + parser.add_argument('task_name', type=str) + now_task = None + + # Get task information based on task name + try: + task_name = parser.parse_args().task_name.upper() + exec(f'now_task = {task_name}') + except: + raise ValueError("The task name is wrong.") + + # Run main function with task information + main(now_task) + + + +""" +Usage: +python code_gen/task_generation.py task_name +""" diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/task_generation_mm.py b/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/task_generation_mm.py new file mode 100644 index 00000000..cc665b7c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/task_generation_mm.py @@ -0,0 +1,357 @@ +import os +import sys +import json + +# Add the project root directory to the path +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from gpt_agent import * +from prompt import * +from task_info import * +from observation_agent import * +from test_gen_code import * + +import argparse +import os + + +def generate_code(task_info, las_error=None, observation_feedback=None, message:list=None, generate_num_id=None): + # Extract task information + if message is None: + message = [] + + # Extract task information + task_name = task_info['task_name'] + task_description = task_info['task_description'] + current_code = task_info['current_code'] + + # Get the enriched actor list + original_actor_list = task_info['actor_list'] + actor_list = enrich_actors(original_actor_list) + + available_env_function = str(AVAILABLE_ENV_FUNCTION) + function_example = str(FUNCTION_EXAMPLE) + + # Generate code + if las_error is not None: + # Include multimodal observation feedback + if observation_feedback: + Prompt = ( + f"The code is unsuccessful, \n# Last Error Message: \n{las_error}\n\n" + f"# Visual Observation Feedback: \n{observation_feedback}\n\n" + f"# Task Description: \n{task_description}\n\n" + f"# Actor List: \n{actor_list}\n\n" + ) + else: + Prompt = ( + f"The code is unsuccessful, \n# Last Error Message: \n{las_error}\n\n" + f"# Task Description: \n{task_description}\n\n" + f"# Actor List: \n{actor_list}\n\n" + ) + else: + res = f''' +from envs._base_task import Base_Task +from envs.{task_name} import {task_name} +from envs.utils import * +import sapien + +class gpt_{task_name}({task_name}): + def play_once(self): + pass + ''' + file_name = f"envs_gen/gpt_{task_name}.py" + with open(file_name, 'w', encoding='utf-8') as file: + file.write(res) + + # Construct the full prompt with all required information + Prompt = ( + f"{BASIC_INFO}\n\n" + f"# Task Description: \n{task_description}\n\n" + f"# Actor List: \n{actor_list}\n\n" + f"# Available API: \n{available_env_function}\n\n" + f"# Function Example: \n{function_example}\n\n" + f"# Current Code:\n{current_code}" + ) + message.append({"role": "user", "content": Prompt}) + + # Start the generation process + res = generate(message) + res = f''' +from envs._base_task import Base_Task +from envs.{task_name} import {task_name} +from envs.utils import * +import sapien + +class gpt_{task_name}({task_name}): + ''' + res[res.find('def play_once'):res.rfind("```")] + + # Save the original code for later comparison + original_code = res + + analysis_text = "" # Initialize analysis text + + # Insert observation function regardless of error + observation_output = insert_observation_points(task_info, res, generate_num_id=generate_num_id) + print("Observation Output: ", observation_output) + + # Extract analysis text (if exists) + if "# task_step:" in observation_output: + try: + step_part = observation_output.split("# task_step:")[1] + if "# task_code:" in step_part: + analysis_text = step_part.split("# task_code:")[0].strip() + except: + print("Error extracting analysis text") + + # Extract the modified code part + if "# task_code:" in observation_output: + code_parts = observation_output.split("# task_code:") + if len(code_parts) > 1: + code_part = code_parts[1].strip() + + # Handle possible markdown code block format + if "```python" in code_part: + code_content = code_part.split("```python", 1)[1] + if "```" in code_content: + code_content = code_content.split("```", 1)[0] + res = code_content.strip() + elif "```" in code_part: + code_content = code_part.split("```", 1)[1] + if "```" in code_content: + code_content = code_content.split("```", 1)[0] + res = code_content.strip() + else: + res = code_part + + # Add analysis text as a comment at the end of the code + if analysis_text: + formatted_analysis = "\n\n'''\nObservation Point Analysis:\n" + analysis_text + "\n'''\n" + res = res + formatted_analysis + + file_name = f"envs_gen/gpt_{task_name}.py" + with open(file_name, 'w', encoding='utf-8') as file: + file.write(res) + + print("Task Name: ", task_name) + print("Task Description: ", task_description) + + task, args = setup_task_config(task_name) + + try: + # Update this to match the new return values of run() + success_rate, error_message, error_count, run_records = run(task, args) + + return res, success_rate, error_message, error_count, run_records + except KeyboardInterrupt: + print("Testing interrupted by user") + return res, 0, "Testing interrupted by user", 20, [] + except Exception as e: + import traceback + error_trace = traceback.format_exc() + print(f"Error occurred during testing: {e}\n{error_trace}") + return res, 0, f"Error occurred during testing: {e}", 20, [] + + +def main(task_info_dic): + # Keys: "task_name", "task_description", "current_code" + + task_info = now_task_info = task_info_dic + messages=[{"role": "system", "content": "You need to generate relevant code for some robot tasks in a robot simulation environment based on the provided API."}] + generate_num = 5 + success_threshold = 0.5 + las_error_message = None + observation_feedback = None + task_name = task_info['task_name'] + task_description = task_info['task_description'] + + # Save the best code and success rate + best_code = None + best_success_rate = 0 + best_run_records = None + + # Create log file + import datetime + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + log_dir = "envs_gen/logs" + os.makedirs(log_dir, exist_ok=True) + log_filename = f"{log_dir}/{task_name}_{timestamp}.log" + + # Store all trial records + all_attempts = [] + suc_list = [] + + # Set the camera image directory path + script_dir = os.path.dirname(os.path.abspath(__file__)) + base_dir = os.path.dirname(script_dir) # Get project root directory + camera_dir = os.path.join(base_dir, "camera_images") + task_camera_dir = os.path.join(camera_dir, task_name.lower()) + + # Clear the camera image directory at the start + def clear_images(directory): + if os.path.exists(directory): + print(f"Clearing image directory: {directory}") + for item in os.listdir(directory): + item_path = os.path.join(directory, item) + try: + if os.path.isdir(item_path): + clear_images(item_path) + print(f"Cleaned directory: {item_path} (directory structure retained)") + else: + image_extensions = ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp'] + file_ext = os.path.splitext(item_path)[1].lower() + + if file_ext in image_extensions: + os.remove(item_path) + print(f"Deleted image: {item_path}") + else: + print(f"Skipped non-image file: {item_path}") + except Exception as e: + print(f"Error processing item {item_path}: {e}") + + clear_images(task_camera_dir) + + for id in range(generate_num): + print("Generate code for task: ", task_name, f"({id+1}/{generate_num})") + + # Generate code + res_code, success_rate, las_error_message, error_count, run_records = generate_code( + now_task_info, + las_error_message, + observation_feedback, + messages, + generate_num_id=id + ) + + suc_list.append(success_rate) + + # Record this attempt + attempt_record = { + "attempt_id": id + 1, + "success_rate": success_rate, + "error_message": las_error_message, + "error_count": error_count, + "code": res_code, + "run_records": run_records + } + all_attempts.append(attempt_record) + + # Save the best code + if success_rate > best_success_rate: + best_success_rate = success_rate + best_code = res_code + best_run_records = run_records + print(f"New best code found with success rate: {best_success_rate}") + + if success_rate >= success_threshold: + print("Successfully generated code for task: ", task_name) + break + + # Handle failure case + print(f"Failed to generate code for task: {task_info['task_name']} {id}\nError message: \n{las_error_message}") + change_info = """The error may be caused by: +1. pre_dis_axis is not set correctly in the place_actor function; +2. the functional point is not set correctly in the place_actor function; +3. The pre_dis or dis is not set correctly in the place_actor function; +4. The constrain is not set correctly in the place_actor function, free or align is not constantly fixed, if the code did not have above error, please try to set the constrain to another value. +5. The code didn't take into account the note given in the example function. +The task can be accomplished only through the existing API and example function, please do not use any other API that is not listed in the available API list and examples.\n""" + now_task_info["task_description"] = f"{task_description}\nFailed to generate code, error message: {las_error_message}, error count: {str(error_count)}\n" + change_info + now_task_info["current_code"] = res_code + + # Analyze run_records to decide which failure case to observe + print("Analyzing run records to determine which error to observe...") + + # Define error priorities + error_list = [ + "The code can not run", + "The target position of the object is incorrect.", + "The left arm failed to grasp the object", + "The right arm failed to grasp the object", + "Plan execution failed", + "Unknown error occurred during execution" + ] + + observe_index = 0 + highest_priority = len(error_list) + + for i, record in enumerate(run_records): + if record == "success!": + continue + + current_priority = len(error_list) + for p, error_pattern in enumerate(error_list): + if error_pattern in record: + current_priority = p + break + + if current_priority < highest_priority: + highest_priority = current_priority + observe_index = i + + if highest_priority == len(error_list) and len(run_records) > 0: + observe_index = 0 + + print(f"Selected to observe error at index {observe_index}: {run_records[observe_index]}") + + # Get multimodal observation feedback + print(f"Selected observation index observe_index={observe_index}, corresponding error: {run_records[observe_index]}") + generate_specific_dir = os.path.join(camera_dir, task_name.lower(), f"generate_num_{id}") + print(f"Looking for images in: {os.path.abspath(generate_specific_dir)}") + observation_feedback = observe_task_execution( + episode_id=observe_index, + task_name=f"{task_name}", + task_info={ + "description": task_info["task_description"], + "goal": "Successfully execute the robot task" + }, + problematic_code=res_code, + save_dir=os.path.dirname(generate_specific_dir), + generate_dir_name=f"generate_num_{id}" + ) + print("Observation feedback: ", observation_feedback) + print("Observation feedback collected") + + # Ensure the best code is saved + if best_code is not None: + file_name = f"envs_gen/gpt_{task_name}.py" + print(f"Saving best code with success rate: {best_success_rate}") + with open(file_name, 'w', encoding='utf-8') as file: + file.write(best_code) + + # Save log information to file + with open(log_filename, 'w', encoding='utf-8') as log_file: + log_data = { + "task_name": task_name, + "task_description": task_info['task_description'], + "best_success_rate": best_success_rate, + "success_rates": suc_list, + "best_code": best_code, + "best_run_records": best_run_records, + "all_attempts": all_attempts + } + json.dump(log_data, log_file, indent=2) + + print("Success rate list: ", suc_list) + print(f"Best success rate: {best_success_rate}") + print(f"Log saved to: {log_filename}") + + return best_success_rate, suc_list, best_code, best_run_records + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Process some integers.') + parser.add_argument('task_name', type=str) + now_task = None + + try: + task_name = parser.parse_args().task_name.upper() + exec(f'now_task = {task_name}') + except Exception as e: + raise ValueError(f"The task name is wrong: {e}") + + main(now_task) + + +""" +Usage: +python code_gen/task_generation_mm.py task_name +""" diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/task_generation_simple.py b/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/task_generation_simple.py new file mode 100644 index 00000000..26ac4a6e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/task_generation_simple.py @@ -0,0 +1,103 @@ +import sys +import os +import json + +# Add project root directory to system path +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from gpt_agent import * +from prompt import * +from task_info import * +from test_gen_code import * +import argparse + +def generate_code_once(task_info): + # Extract task information + task_name = task_info['task_name'] + task_description = task_info['task_description'] + current_code = task_info['current_code'] + + # Get the enriched actor_list + original_actor_list = task_info['actor_list'] + actor_list = enrich_actors(original_actor_list) + + available_env_function = str(AVAILABLE_ENV_FUNCTION) + function_example = str(FUNCTION_EXAMPLE) + + # Construct prompt + prompt = ( + f"{BASIC_INFO}\n\n" + f"# Task Description:\n{task_description}\n\n" + f"# Actor List:\n{actor_list}\n\n" + f"# Available API:\n{available_env_function}\n\n" + f"# Function Example:\n{function_example}\n\n" + f"# Current Code:\n{current_code}" + ) + + message = [ + {"role": "system", "content": "You need to generate relevant code for some robot tasks in a robot simulation environment based on the provided API."}, + {"role": "user", "content": prompt} + ] + + # Generate code from model + res = generate(message, gpt="deepseek", temperature=0) + + # Extract the relevant portion of the generated code + res = f''' +from envs._base_task import Base_Task +from envs.{task_name} import {task_name} +from envs.utils import * +import sapien + +class gpt_{task_name}({task_name}): + ''' + res[res.find('def play_once'):res.rfind("```")] + + # Save to file + file_name = f"envs_gen/gpt_{task_name}.py" + os.makedirs(os.path.dirname(file_name), exist_ok=True) + with open(file_name, 'w') as f: + f.write(res) + + return res + + +def main(task_info): + print("Generating code once for task:", task_info['task_name']) + code = generate_code_once(task_info) + + print("Generated code saved. Testing...") + + task, args = setup_task_config(task_info['task_name']) + + try: + success_rate, error_message, error_count, run_records = run(task, args) + print(f"Success Rate: {success_rate}") + print("Run Records:", run_records) + except Exception as e: + import traceback + print("Error during run:") + print(traceback.format_exc()) + success_rate, error_message, error_count, run_records = 0, str(e), 1, None + + return code, success_rate, error_message, error_count, run_records + + + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Process some integers.') + parser.add_argument('task_name', type=str) + now_task = None + + try: + task_name = parser.parse_args().task_name.upper() + exec(f'now_task = {task_name}') + except Exception as e: + raise ValueError(f"The task name is wrong: {e}") + + main(now_task) + +""" +Usage: +python code_gen/task_generation_simple.py task_name +""" diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/task_info.py b/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/task_info.py new file mode 100644 index 00000000..b7772115 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/task_info.py @@ -0,0 +1,1431 @@ +# All variable names for task information must be in uppercase. + +# Template of Task Information: +""" +TASK_NAME = { + "task_name": "task_name", # Name of the task + "task_description": "...", # Detailed description of the task + "current_code": ''' + class gpt_{task_name}({task_name}): + def play_once(self): + pass + ''' # Code template to be completed + "actor_list": { # List of involved objects; can be a dictionary or a simple list + "self.object1": { + "name": "object1", # Object name + "description": "...", # Description of the object + "modelname": "model_name" # Name of the 3D model representing the object + }, + "self.object2": { + "name": "object2", + "description": "...", + "modelname": "model_name" + }, + # ... more objects + }, + # Alternatively, the actor_list can be a simple list: + # "actor_list": ["self.object1", "self.object2", ...], + # To make Code Generation easier, the actor_list also includes some pose like target pose or middle pose, this is optional and dont have modelname. +} +""" + +################## Known Tasks ################## +# These tasks are used to debug and iterate on prompt design. +# Prompt instructions have been specifically adjusted for them. + + +BEAT_BLOCK_HAMMER = { + "task_name": "beat_block_hammer", + "task_description": + "Pick up the hammer and use it to beat the block on the table once. The hammer is placed at a fixed position on the table, \ + but the block is generated randomly on the table. If the block's x coordinate (dim 0) is greater than 0, use the right arm to grasp the hammer, \ + else use the left arm. To beat the block, you should place the hammer on the block's functional point \ + (i.e., use the place_actor API to align the hammer's contact point with the block's functional point). \ + Note: You don't need to Lift the hammer after beating the block, and you don't need to open the gripper or return the arm to origin position.", + "current_code": """ + class gpt_beat_block_hammer(beat_block_hammer): + def play_once(self): + pass + """, + "actor_list": { + "self.hammer": { + "name": "hammer", + "description": "The hammer used to beat the block.", + "modelname": "020_hammer" + }, + "self.block": { + "name": "block", + "description": "The block that needs to be beaten by the hammer.", + "modelname": "sapien-block1", + } + }, +} + +PICK_DUAL_BOTTLES_EASY = { + "task_name": "pick_dual_bottles", + "task_description": + "Use both arms to simultaneously pick up the bottle1 and bottle2 and move them to the front target locations, \ + with the bottle1 on the left and the bottle2 on the right.\ + Note: You don't need to open gripper and don't put down the bottles at the end.", + "current_code": """ + class gpt_pick_dual_bottles(pick_dual_bottles): + def play_once(self): + pass + """, + "actor_list": { + "self.bottle1": { + "name": "bottle1", + "description": "The first bottle to be picked up, placed on the left side.", + "modelname": "001_bottle", + }, + "self.bottle2": { + "name": "bottle2", + "description": "The second bottle to be picked up, placed on the right side.", + "modelname": "001_bottle", + }, + "self.left_target_pose": { + "name": "left_target_pose", + "description": "The target pose for the left arm to place bottle1.", + "modelname": None, + }, + "self.right_target_pose": { + "name": "right_target_pose", + "description": "The target pose for the right arm to place bottle2.", + "modelname": None, + } + }, +} + +PICK_DIVERSE_BOTTLES = { + "task_name": "pick_diverse_bottles", + "task_description": + "Use both arms to simultaneously pick up the diverse bottles and move them to the front target locations, \ + with the bottle1 on the left and the bottle2 on the right. No need to put the bottles down. In which the bottles may be lying down.\ + Note: You don't need to open gripper and don't put down the bottles at the end.", + "current_code": """ + class gpt_pick_diverse_bottles(pick_diverse_bottles): + def play_once(self): + pass + """, + "actor_list": { + "self.bottle1": { + "name": "bottle1", + "description": "The first bottle to be picked up, placed on the left side.", + "modelname": "001_bottle", + }, + "self.bottle2": { + "name": "bottle2", + "description": "The second bottle to be picked up, placed on the right side.", + "modelname": "001_bottle", + }, + "self.left_target_pose": { + "name": "left_target_pose", + "description": "The target pose for the left arm to place bottle1.", + "modelname": None, + }, + "self.right_target_pose": { + "name": "right_target_pose", + "description": "The target pose for the right arm to place bottle2.", + "modelname": None, + } + }, +} + +HANDOVER_BLOCK = { + "task_name": "handover_block", + "task_description": + "There are two blocks on the desk. Use the left arm to grab the block and move it to the handover point, then use right arm to grab the block and open the gripper of left arm simutaniously.\ + Use right arm move block on the target block.\ + Note: You should first pass the block to the right gripper and close right gripper, then open the left gripper.", + "current_code": """ + class gpt_handover_block(handover_block): + def play_once(self): + pass + """, + "actor_list": { + "self.box": { + "name": "box", + "description": "The red long box that needs to be handed over.", + "modelname": "sapien-block2", + }, + "self.target_box": { + "name": "target_box", + "description": "The blue target box where the red box needs to be placed.", + "modelname": "sapien-block1", + }, + "self.block_middle_pose": { + "name": "block_middle_pose", + "description": "The middle pose where the block needs to be handed over.", + "modelname": None, + } + }, +} + +STACK_BLOCKS_TWO = { + "task_name": "stack_blocks_two", + "task_description": + "Use the gripper to pick up block1 and move block 1 to the target position. Then pick up block 2 and place it on the block 1.\ + If block1's x coordinate (dim 0) is greater than 0, use right arm to stack the block1, else use the left arm, and same for the block2.\ + Note: You need to call the get_avoid_collision_pose function to avoid collisions when the left and right arms move alternately. \ + For example, if the previous action uses the left arm and the next action uses the right arm, you need to move the left arm after release gripper to avoid collisions, vice versa.\ + The pre-dis of stacked blocks may be smaller.", + "current_code": """ + class gpt_stack_blocks_two(stack_blocks_two): + def play_once(self): + pass + """, + "actor_list": { + "self.block1": { + "name": "block1", + "description": "The first block to be stacked.", + "modelname": "sapien-block1", + }, + "self.block2": { + "name": "block2", + "description": "The second block to be stacked on top of the first block.", + "modelname": "sapien-block1", + }, + "self.block1_target_pose": { + "name": "block1_target_pose", + "description": "The target pose for the first block after stacking." + } + }, +} + +STACK_BLOCKS_THREE = { + "task_name": "stack_blocks_three", + "task_description": + "Use the gripper to pick up block1 and move block 1 to the target position. Then pick up block 2 and place it on the block 1, and finally pick up\ + block3 and place it on the block2.\ + If block1's x coordinate (dim 0) is greater than 0, use right arm to stack the block1, else use the left arm. And same for the block2 and block3.\ + Note: You need to call the get_avoid_collision_pose function to avoid collisions when the left and right arms move alternately. \ + For example, if the previous action uses the left arm and the next action uses the right arm, you need to move the left arm after release gripper to avoid collisions, vice versa.\ + The pre-dis of stacked blocks may be smaller.", + "current_code": """ + class gpt_stack_blocks_three(stack_blocks_three): + def play_once(self): + pass + """, + "actor_list": { + "self.block1": { + "name": "block1", + "description": "The first block to be stacked.", + "modelname": "sapien-block1", + }, + "self.block2": { + "name": "block2", + "description": "The second block to be stacked on top of the first block.", + "modelname": "sapien-block1", + }, + "self.block3": { + "name": "block3", + "description": "The third block to be stacked on top of the second block.", + "modelname": "sapien-block1", + }, + "self.block1_target_pose": { + "name": "block1_target_pose", + "description": "The target pose for the first block after stacking." + } + }, +} + +PLACE_CONTAINER_PLATE = { + "task_name": "place_container_plate", + "task_description": + "Use both arms to pick up the container and place it in the plate. If the container's x coordinate (dim 0) is greater than 0, \ + use right arm to grasp the right side of the container, then pick up the container and place it in the plate. \ + Else use the left arm grasp the left side of the container, then pick up the container and place it in the plate.\ + Note: You may need to close the jaws tightly to pick up the container.", + "current_code": """ + class gpt_place_container_plate(place_container_plate): + def play_once(self): + pass + """, + "actor_list": { + "self.container": { + "name": "container", + "description": "The container that needs to be placed in the plate.", + "modelname": "002_bowl", + }, + "self.plate": { + "name": "plate", + "description": "The plate where the container needs to be placed.", + "modelname": "003_plate", + } + }, +} + +PLACE_EMPTY_CUP = { + "task_name": "place_empty_cup", + "task_description": + "Use both arms to pick up the empty cup and place it on the coaster. If the cup's x coordinate (dim 0) is greater than 0, \ + use right arm to grasp the cup, then pick up the cup and place it on the coaster,\ + else use the left arm grasp the the cup, then pick up the cup and place it on the coaster.\ + Note: You may need to close the jaws tightly to pick up the cup.\ + Pre-dis for grabbing and placing cups may be smaller.\ + The distance of lifting the cup may be smaller.", + "current_code": """ + class gpt_place_empty_cup(place_empty_cup): + def play_once(self): + pass + """, + "actor_list": { + "self.cup": { + "name": "cup", + "description": "The empty cup that needs to be placed on the coaster.", + "modelname": "021_cup", + }, + "self.coaster": { + "name": "coaster", + "description": "The coaster where the empty cup needs to be placed.", + "modelname": "019_coaster", + } + }, +} + +PLACE_SHOE = { + "task_name": "place_shoe", + "task_description": + "Pick up the shoe and place it on the target block. And the head of the shoe should be towards the left side.\ + The shoe is randomly placed on the table, if the shoe's x coordinate (dim 0) is greater than 0, use right arm to grasp the shoe, \ + else use the left arm grasp the shoe.", + "current_code": """ + class gpt_place_shoe(place_shoe): + def play_once(self): + pass + """, + "actor_list": { + "self.shoe": { + "name": "shoe", + "description": "The shoe that needs to be placed on the target block.", + "modelname": "041_shoe", + }, + "self.target_block": { + "name": "target_block", + "description": "The target block where the shoe needs to be placed.", + "modelname": "sapien-block1", + } + }, +} + +PLACE_DUAL_SHOES = { + "task_name": "place_dual_shoes", + "task_description": + "Left shoe and right shoe are randomly generated on the desktop, one on the left and one on the right.\ + Use left and right arms to pick up two shoes simultaneously. And put down them on the shoe box respectively.\ + The head of the shoe should be towards the left side.\ + Left shoe should be placed on the point0 of shoe box, and right shoe should be placed on the point1 of shoe box.\ + Note: You may need to put the shoes in order to avoid left and right arm collisions.\ + Avoiding collisions needs to be done before place shoes.\ + Pre-dis for grabbing and placing shoes may be smaller.", + "current_code": """ + class gpt_place_dual_shoes(place_dual_shoes): + def play_once(self): + pass + """, + "actor_list": { + "self.left_shoe": { + "name": "left_shoe", + "description": "The left shoe that needs to be placed in the shoe box.", + "modelname": "041_shoe", + }, + "self.right_shoe": { + "name": "right_shoe", + "description": "The right shoe that needs to be placed in the shoe box.", + "modelname": "041_shoe", + }, + "self.shoe_box": { + "name": "shoe_box", + "description": "The shoe box where the shoes need to be placed.", + "modelname": "007_shoe-box", + }, + "self.right_shoe_middle_pose": { + "name": "right_shoe_middle_pose", + "description": + "The middle pose for the right arm to place the right shoe, which is used to avoid collisions when placing the left shoes.", + "modelname": None, + } + }, +} + + +################## Generalization Test Tasks ################## +# These tasks are used to evaluate the generalization ability of the code generation. +# No task-specific prompt tuning has been applied to them. + + +ADJUST_BOTTLE = { + "task_name": "adjust_bottle", + "task_description": "Pick up the bottle on the table headup with the correct arm.\ + Move the arm upward by 0.1 meters along z-axis, and place the bottle at target pose.\ + Note: You should keep gripper closed when placing the bottle.", + "current_code": """ + class gpt_adjust_bottle(adjust_bottle): + def play_once(self): + pass + """, + "actor_list": { + "self.bottle": { + "name": "bottle", + "description": "The bottle should be picked up and placed at the target pose.", + "modelname": "001_bottle" + }, + "self.qpose_tag": { + "name": "qpose_tag", + "description": "A tag indicating which arm to use for picking up the bottle.\ + 0 means left arm, 1 means right arm.", + }, + "self.left_target_pose": { + "name": "left_target_pose", + "description": "Target pose when use left arm to pick up the bottle.", + }, + "self.right_target_pose": { + "name": "right_target_pose", + "description": "Target pose when use right arm to pick up the bottle.", + } + } +} + +BLOCKS_RANKING_RGB= { + "task_name": "blocks_ranking_rgb", + "task_description": "Place the red block, green block, and blue block in the order of red, green, and blue from left to right, placing in a row.\ + Pick and place each block to their target positions.\ + Note: You should move end effector back to origin after placing each block to avoid collisions.\ + You can place the red block, the green block, and the blue block in the order.", + "current_code": """ + class gpt_blocks_ranking_rgb(blocks_ranking_rgb): + def play_once(self): + pass + """, + "actor_list": { + "self.block1": { + "name": "box", + "description": "Red block that should be placed on the left side.", + "modelname": "sapien-block1", + }, + "self.block2": { + "name": "box", + "description": "Green block that should be placed in the middle.", + "modelname": "sapien-block1", + }, + "self.block3": { + "name": "box", + "description": "Blue block that should be placed on the right side.", + "modelname": "sapien-block1", + }, + "self.block1_target_pose": { + "name": "target_pose", + "description": "Target pose for the red block.", + }, + "self.block2_target_pose": { + "name": "target_pose", + "description": "Target pose for the green block.", + }, + "self.block3_target_pose": { + "name": "target_pose", + "description": "Target pose for the blue block.", + } + } +} + +BLOCKS_RANKING_SIZE = { + "task_name": "blocks_ranking_size", + "task_description": "There are three blocks on the table, the color of the blocks is random, move the blocks to the center of the table, and arrange them from largest to smallest, from left to right.\ + Pick and place each block to their target positions.\ + Note: You should move end effector back to origin after placing each block to avoid collisions.\ + You can place the smallest block, the middle block, and the largest block in the order.", + "current_code": """ + class gpt_blocks_ranking_size(blocks_ranking_size): + def play_once(self): + pass + """, + "actor_list": { + "self.block1": { + "name": "box", + "description": "The largest block that should be placed on the left side.", + "modelname": "sapien-block1", + }, + "self.block2": { + "name": "box", + "description": "The middle block that should be placed in the middle.", + "modelname": "sapien-block1", + }, + "self.block3": { + "name": "box", + "description": "The smallest block that should be placed on the right side.", + "modelname": "sapien-block1", + }, + "self.block1_target_pose": { + "name": "target_pose", + "description": "Target pose for the largest block.", + }, + "self.block2_target_pose": { + "name": "target_pose", + "description": "Target pose for the middle block.", + }, + "self.block3_target_pose": { + "name": "target_pose", + "description": "Target pose for the smallest block.", + } + } +} + +CLICK_BELL = { + "task_name": "click_bell", + "task_description": "Click the bell's top center on the table.\ + Move the top of bell's center and close gripper. And move the gripper down to touch the bell's top center.\ + Note: You can change some API parameters to move above the bell's top center and close the gripper.\ + You can use self.grasp_actor() to simulate the action of touch and click.\ + self.grasp_actor() is only used to move the top center of the bell and close the gripper. So you must use same pre_grasp_dis and grasp_dis as the click_bell task.\ + You don't need to lift the bell after clicking it, and you don't need to open the gripper or return the arm to origin position.", + "current_code": """ + class gpt_click_bell(click_bell): + def play_once(self): + pass + """, + "actor_list": { + "self.bell": { + "name": "bell", + "description": "The bell that needs to be clicked.", + "modelname": "050_bell", + } + } +} + +GRAB_ROLLER = { + "task_name": "grab_roller", + "task_description": "Use both arms to grab the roller on the table.\ + Grasp the roller with both arms simultaneously at different contact points.\ + And lift the roller upper by 0.15 meters by moving both arms upward simultaneously.", + "current_code": """ + class gpt_grab_roller(grab_roller): + def play_once(self): + pass + """, + "actor_list": { + "self.roller": { + "name": "roller", + "description": "The roller that needs to be grabbed.", + "modelname": "102_roller", + } + } +} + +LIFT_POT = { + "task_name": "lift_pot", + "task_description": "Use both arms to lift the pot.\ + Grasp the pot with both arms at specified contact points. And lift the pot by moving both arms upper by 0.15 meters.\ + Note: The pre_grasp_dis is very small when grasping the pot.", + "current_code": """ + class gpt_lift_pot(lift_pot): + def play_once(self): + pass + """, + "actor_list": { + "self.pot": { + "name": "pot", + "description": "The pot that needs to be lifted.", + "modelname": "060_kitchenpot", + } + } +} + +MOVE_CAN_POT = { + "task_name": "move_can_pot", + "task_description": "There is a can and a pot on the table. Use one arm to pick up the can and move it to beside the pot.\ + Grasp the can, and move the can upward. Place the can near the pot at target pose.\ + Note: You don't need to return the arm to origin position. ", + "current_code": """ + class gpt_move_can_pot(move_can_pot): + def play_once(self): + pass + """, + "actor_list": { + "self.can": { + "name": "can", + "description": "The can that needs to be moved to the pot.", + "modelname": "105_sauce-can", + }, + "self.pot": { + "name": "pot", + "description": "The pot at the center of the table.", + "modelname": "060_kitchenpot", + }, + "self.target_pose":{ + "name": "target_pose", + "description": "The target pose where the can should be placed beside the pot.", + } + } +} + +MOVE_PLAYINGCARD_AWAY = { + "task_name": "move_playingcard_away", + "task_description": "Use the arm to pick up the playing card and move it to left or right.\ + Grasp the playing cards with specified arm, and then move the playing cards horizontally (right if right arm, left if left arm).\ + Note: You should open gripper to release the playing cards after moving them.", + "current_code": """ + class gpt_move_playingcard_away(move_playingcard_away): + def play_once(self): + pass + """, + "actor_list": { + "self.playingcards": { + "name": "playingcards", + "description": "The playing cards that need to be moved to left or right.", + "modelname": "081_playingcards", + } + } +} + +MOVE_STAPLER_PAD = { + "task_name": "move_stapler_pad", + "task_description": "Use appropriate arm to move the stapler to a colored mat.\ + Grasp the stapler with specified arm, and move the arm upward. Place the stapler at target pose with alignment constraint.", + "current_code": """ + class gpt_move_stapler_pad(move_stapler_pad): + def play_once(self): + pass + """, + "actor_list": { + "self.stapler": { + "name": "stapler", + "description": "The stapler that needs to be moved to the pad.", + "modelname": "048_stapler", + }, + "self.target_pose": { + "name": "target", + "description": "The target pose where the stapler should be placed on the pad." + } + } +} + + +CLICK_ALARMCLOCK = { + "task_name": "click_alarmclock", + "task_description": "Click the alarm clock's center of the top side button on the table.\ + Move the top of bell's center and close gripper. And move the gripper down.\ + Note: You can change some API parameters to move above the alarm clock's top center and close the gripper(grasp_actor).\ + You can use self.grasp_actor() to simulate the action of touch and click", + "current_code": """ + class gpt_click_alarmclock(click_alarmclock): + def play_once(self): + pass + """, + "actor_list": { + "self.alarm": { + "name": "alarm", + "description": "The alarm clock that needs to be clicked.", + "modelname": "046_alarm-clock", + } + } +} + +DUMP_BIN_BIGBIN = { + "task_name": "dump_bin_bigbin", + "task_description": "Grab the small bin and pour the balls into the big bin.\ + If the small bin is on the right side of the table, grasp the deskbin with right arm, and place the deskbin at middle pose.\ + Then return right arm to origin while simultaneously grasping with left arm. If deskbin is on left side, directly grasp with left arm.\ + Perform pouring actions 3 times.\ + Note: The gripper should be closed when pouring the balls into the big bin.\ + You should use self.delay(6) in the end of the task to wait for the pouring actions to complete.\ + Don't use functional point for the deskbin, use self.place_actor() to place the deskbin at middle pose.", + "current_code": """ + class gpt_dump_bin_bigbin(dump_bin_bigbin): + def play_once(self): + pass + """, + "actor_list": { + "self.deskbin": { + "name": "deskbin", + "description": "The small bin that needs to be dumped into the big bin.", + "modelname": "005_desk-bin", + }, + "self.pour_actions": { + "name": "pour_actions", + "description": "The final actions for pouring the balls into the big bin, use self.move(self.pour_actions) to perform the pouring actions.", + "modelname": None, + }, + "self.middle_pose": { + "name": "middle_pose", + "description": "The middle pose where the deskbin should be placed before pouring.", + "modelname": None, # No specific model for this pose + } + } +} + +HANDOVER_MIC = { + "task_name": "handover_mic", + "task_description": "Use one arm to grasp the microphone on the table and handover it to the other arm.\ + Move the grasping arm to the microphone's position and grasp it. Move the handover arm to the middle position for handover. \ + Move the handover arm to grasp the microphone from the grasping arm. Move the grasping arm to open the gripper.", + "current_code": """ + class gpt_handover_mic(handover_mic): + def play_once(self): + pass + """, + "actor_list": { + "self.microphone": { + "name": "microphone", + "description": "The microphone that needs to be handed over.", + "modelname": "018_microphone", + }, + "self.handover_middle_pose": { + "name": "handover_middle_pose", + "description": "The middle pose for the handover arm to grasp the microphone from the grasping arm.", + "modelname": None, # No specific model for this pose + } + } +} + +HANGING_MUG = { + "task_name": "hanging_mug", + "task_description": "Use left arm to pick the mug on the table, rotate the mug and put the mug down in the middle of the table, use the right arm to pick the mug and hang it onto the rack.\ + Move the grasping arm to the mug's position and grasp it. Move the grasping arm to a middle position before hanging.\ + Grasp the mug with the hanging arm, and move the grasping arm back to its origin. Move the hanging arm to the target pose and hang the mug.\ + Note: grasping arm is left arm, hanging arm is right arm.\ + The target pose for hanging the mug is the functional point of the rack.", + "current_code": """ + class gpt_hanging_mug(hanging_mug): + def play_once(self): + pass + """, + "actor_list": { + "self.mug": { + "name": "mug", + "description": "The mug that needs to be hung onto the rack.", + "modelname": "039_mug", + }, + "self.rack": { + "name": "rack", + "description": "The rack where the mug should be hung.", + "modelname": "040_rack", + }, + "self.middle_pos": { + "name": "middle_pos", + "description": "The middle position where the grasping arm should be moved before hanging the mug.", + "modelname": None, # No specific model for this pose + } + } +} + + +MOVE_PILLBOTTLE_PAD = { + "task_name": "move_pillbottle_pad", + "task_description": "Use one arm to pick the pillbottle and place it onto the pad.\ + Grasp the pillbottle. Get the target pose for placing the pillbottle, and place the pillbottle at the target pose.", + "current_code": """ + class gpt_move_pillbottle_pad(move_pillbottle_pad): + def play_once(self): + pass + """, + "actor_list": { + "self.pillbottle": { + "name": "pillbottle", + "description": "The pillbottle that needs to be moved to the pad.", + "modelname": "080_pillbottle", + }, + "self.pad": { + "name": "pad", + "description": "The pad where the pillbottle should be placed.", + "modelname": "sapien-block1", + }, + } +} + +PLACE_A2B_LEFT = { + "task_name": "place_a2b_left", + "task_description": "Use appropriate arm to place object on the left of target object.\ + Grasp the object with specified arm. And get target pose and adjust x position to place object to the left of target object.\ + Place the object at the adjusted target object position.\ + Note: You can decrease the x position of target pose by 0.13 to place object to the left of target object. (target_pose[0] -= 0.13)", + "current_code": """ + class gpt_place_a2b_left(place_a2b_left): + def play_once(self): + pass + """, + "actor_list": { + "self.object": { + "name": "object", + "description": "The object that needs to be placed on the left of the target object.", + "modelname": None, # Replace with actual model name + }, + "self.target_object": { + "name": "target_object", + "description": "The target object where the object should be placed to its left, you can get the target pose from this object by target_pose = self.target_object.get_pose().p.tolist()", + "modelname": None, # Replace with actual model name + }, + } +} + +PLACE_A2B_RIGHT = { + "task_name": "place_a2b_right", + "task_description": "Use appropriate arm to place object on the right of target object.\ + Grasp the object with specified arm. And get target pose and adjust x position to place object to the right of target object.\ + Place the object at the adjusted target object position.\ + Note: You can increase the x position of target pose by 0.13 to place object to the right of target object. (target_pose[0] += 0.13)", + "current_code": """ + class gpt_place_a2b_right(place_a2b_right): + def play_once(self): + pass + """, + "actor_list": { + "self.object": { + "name": "object", + "description": "The object that needs to be placed on the right of the target object.", + "modelname": None, # Replace with actual model name + }, + "self.target_object": { + "name": "target_object", + "description": "The target object where the object should be placed to its left, you can get the target pose from this object by target_pose = self.target_object.get_pose().p.tolist()", + "modelname": None, # Replace with actual model name + }, + } +} + +PLACE_BREAD_BASKET = { + "task_name": "place_bread_basket", + "task_description": "If there is one bread on the table, use one arm to grab the bread and put it in the basket. If there are two breads on the table, use two arms to simultaneously grab up two breads and put them in the basket.\ + Grasp the bread. If there is one bread, place the bread into the basket. If there is two breads, place left bread into the basket, and place right bread into the basket when move left arm back to origin.\ + Note: You should move the arm back to origin after placing the bread to avoid collisions.", + "current_code": """ + class gpt_place_bread_basket(place_bread_basket): + def play_once(self): + pass + """, + "actor_list": { + "self.bread[id]": { + "name": "bread[id]", + "description": "A list of breads that need to be placed in the basket. If there is one bread, id=0. If there are two breads, id=0 and id=1.", + "modelname": "075_bread", + }, + "self.breadbasket": { + "name": "breadbasket", + "description": "The basket where the bread needs to be placed.", + "modelname": "076_breadbasket", + }, + } +} + +PLACE_BREAD_SKILLET = { + "task_name": "place_bread_skillet", + "task_description": "If there is one bread on the table, use one arm to grab the bread and put it into the skillet.\ + Grasp the skillet and bread simultaneously with dual arms. Get the functional point of the skillet as placement target for the bread.\ + Place the bread onto the skillet.", + "current_code": """ + class gpt_place_bread_skillet(place_bread_skillet): + def play_once(self): + pass + """, + "actor_list": { + "self.bread": { + "name": "bread", + "description": "The bread that needs to be placed in the skillet.", + "modelname": "075_bread", + }, + "self.skillet": { + "name": "skillet", + "description": "The skillet where the bread needs to be placed.", + "modelname": "106_skillet", + } + } +} + +PLACE_CAN_BASKET = { + "task_name": "place_can_basket", + "task_description": "Use one arm to pick up the can and place it into the basket. Use the other arm to lift up the basket.\ + Grasp the can with the specified arm. Place the can at the selected position into the basket. Lift the basket with the opposite arm.\ + Note: You should not open the gripper after lifting the basket.\ + The height of lifting the basket is 5 cm.", + "current_code": """ + class gpt_place_can_basket(place_can_basket): + def play_once(self): + pass + """, + "actor_list": { + "self.can": { + "name": "can", + "description": "The can that needs to be placed in the basket.", + "modelname": "071_can", + }, + "self.basket": { + "name": "basket", + "description": "The basket where the can needs to be placed.", + "modelname": "110_basket", + }, + "self.get_arm_pose(arm_tag=self.arm_tag)": { + "name": "place_pose", + "description": "The target pose where the can should be placed in the basket.", + "modelname": None, + } + } +} + +PLACE_CANS_PLASTICBOX = { + "task_name": "place_cans_plasticbox", + "task_description": "Use dual arm to pick and place cans into plasticbox.\ + Grasp both objects with dual arms. Place left object into plastic box at target point 1,\ + and then left arm moves back to origin while right arm places object into plastic box at target point 0.\ + Grasp the second can with the right arm and place it into the plastic box at target point 0. Right arm moves back to original position.\ + Note: You should use left arm to grasp object 1 and right arm to grasp object 2.\ + Don't set pre_dis_axis to fp, because the pre_dis_axis is not used in this task.", + "current_code": """ + class gpt_place_cans_plasticbox(place_cans_plasticbox): + def play_once(self): + pass + """, + "actor_list": { + "self.object1": { + "name": "object1", + "description": "The first object to be placed in the plastic box.", + "modelname": "071_can", + }, + "self.object2": { + "name": "object2", + "description": "The second object to be placed in the plastic box.", + "modelname": "071_can", + }, + "self.plasticbox": { + "name": "plasticbox", + "description": "The plastic box where the objects need to be placed.", + "modelname": "062_plasticbox", + }, + } +} + +PLACE_FAN = { + "task_name": "place_fan", + "task_description": "Grab the fan and place it on a colored pad.\ + Grasp the fan with the selected arm. Place the fan to the target pose.\ + Note: The height of lifting the fan is small. Fan have front and back, so you should use constraint 'align' to align the fan's front with the pad's front.", + "current_code": """ + class gpt_place_fan(place_fan): + def play_once(self): + pass + """, + "actor_list": { + "self.fan": { + "name": "fan", + "description": "The fan that needs to be placed on the pad.", + "modelname": "099_fan", + }, + "self.target_pose": { + "name": "target_pose", + "description": "The target pose where the fan should be placed on the pad.", + "modelname": None, + } + } +} + +PLACE_BURGER_FRIES = { + "task_name": "place_burger_fries", + "task_description": "Use dual arm to pick the hamburg and frenchfries and put them onto the tray.\ + Dual grasp of hamburg and french fries. Get target poses from tray for placing. And place hamburg on tray, then place french fries on tray while moving the arm that placed hamburg back to origin.\ + Note: Use left arm to grasp hamburg and right arm to grasp french fries.\ + The target pose for placing hamburg and french fries is the functional point 0 and 1 of the tray respectively.", + "current_code": """ + class gpt_place_burger_fries(place_burger_fries): + def play_once(self): + pass + """, + "actor_list": { + "self.hamburg": { + "name": "hamburg", + "description": "The hamburg that needs to be placed on the tray.", + "modelname": "006_hamburg", + }, + "self.frenchfries": { + "name": "frenchfries", + "description": "The french fries that needs to be placed on the tray.", + "modelname": "005_french-fries", + }, + "self.tray": { + "name": "tray", + "description": "The tray where the hamburg and french fries need to be placed.", + "modelname": "008_tray", + }, + } +} + +PLACE_MOUSE_PAD = { + "task_name": "place_mouse_pad", + "task_description": "Grasp the mouse and place it on a colored pad.\ + Grasp the mouse with the selected arm. Place the mouse at the target location.\ + Note: The mouse have front and back, so you should use constraint 'align' to align the mouse's front with the pad's front.", + "current_code": """ + class gpt_place_mouse_pad(place_mouse_pad): + def play_once(self): + pass + """, + "actor_list": { + "self.mouse": { + "name": "mouse", + "description": "The mouse that needs to be placed on the pad.", + "modelname": "047_mouse", + }, + "self.target_pose": { + "name": "target_pose", + "description": "The target pose where the mouse should be placed on the pad.", + "modelname": None, + } + } +} + +PLACE_OBJECT_BASKET = { + "task_name": "place_object_basket", + "task_description": "Use one arm to grab the target object and put it in the basket, then use the other arm to grab the basket, and finally move the basket slightly away.\ + Grasp the object with the specified arm. Place the object at the selected position into the basket. Lift the basket with the opposite arm.\ + Note: You should not open the gripper after lifting the basket.\ + The height of lifting the basket is 5 cm.", + "current_code": """ + class gpt_place_object_basket(place_object_basket): + def play_once(self): + pass + """, + "actor_list": { + "self.object": { + "name": "object", + "description": "The object that needs to be placed in the basket.", + "modelname": None, # Replace with actual model name + }, + "self.basket": { + "name": "basket", + "description": "The basket where the object needs to be placed.", + "modelname": "110_basket", + }, + } +} + +PLACE_OBJECT_SCALE = { + "task_name": "place_object_scale", + "task_description": "Use one arm to grab the object and put it on the scale.\ + Grasp the object with the selected arm. Place the object on the scale.\ + Note: Don't use functional_point_id and pre_dis_axis='fp', because the object can be any object that is specified in the task.", + "current_code": """ + class gpt_place_object_scale(place_object_scale): + def play_once(self): + pass + """, + "actor_list": { + "self.object": { + "name": "object", + "description": "The object that needs to be placed on the scale.", + "modelname": None, # The object can be any object that is specified in the task + }, + "self.scale": { + "name": "scale", + "description": "The scale where the object needs to be placed.", + "modelname": "072_electronicscale", + }, + } +} + +PLACE_OBJECT_STAND = { + "task_name": "place_object_stand", + "task_description": "Use appropriate arm to place the object on the stand.\ + Grasp the object with the specified arm. Place the object onto the display stand.\ + Note: Don't use functional_point_id and pre_dis_axis='fp', because the object can be any object that is specified in the task.", + "current_code": """ + class gpt_place_object_stand(place_object_stand): + def play_once(self): + pass + """, + "actor_list": { + "self.object": { + "name": "object", + "description": "The object that needs to be placed on the stand.", + "modelname": None, # The object can be any object that is specified in the task + }, + "self.displaystand": { + "name": "displaystand", + "description": "The display stand where the object needs to be placed.", + "modelname": "074_displaystand", + } + } +} + +PLACE_PHONE_STAND = { + "task_name": "place_phone_stand", + "task_description": "Pick up the phone and put it on the phone stand.\ + Grasp the phone with specified arm. Place the phone onto the stand's functional point and align the points.", + "current_code": """ + class gpt_place_phone_stand(place_phone_stand): + def play_once(self): + pass + """, + "actor_list": { + "self.phone": { + "name": "phone", + "description": "The phone that needs to be placed on the stand.", + "modelname": "077_phone", + }, + "self.stand": { + "name": "stand", + "description": "The phone stand where the phone needs to be placed.", + "modelname": "078_phonestand", + }, + } +} + +PRESS_STAPLER = { + "task_name": "press_stapler", + "task_description": "Use one arm to press the stapler.\ + Move arm to the position of the stapler and close the gripper. Move the stapler down slightly.\ + Note: You can use self.grasp_actor() to simulate the action of move to the position of stapler or pressing the stapler.\ + The stapler should be pressed at the top center.", + "current_code": """ + class gpt_press_stapler(press_stapler): + def play_once(self): + pass + """, + "actor_list": { + "self.stapler": { + "name": "stapler", + "description": "The stapler that needs to be pressed.", + "modelname": "048_stapler", + } + } +} + + +ROTATE_QRCODE = { + "task_name": "rotate_qrcode", + "task_description": "Use arm to catch the qrcode board on the table, pick it up and rotate to let the qrcode face towards you.\ + Grasp the QR code with specified pre-grasp distance. Place the QR code at the target position.\ + Note: The QR code have front and back, so you should use constraint 'align' to align the QR code's front with the target position.\ + Don't use functional point of the QR code when placing it.", + "current_code": """ + class gpt_rotate_qrcode(rotate_qrcode): + def play_once(self): + pass + """, + "actor_list": { + "self.qrcode": { + "name": "qrcode", + "description": "The QR code sign that needs to be rotated.", + "modelname": "070_paymentsign", + }, + "self.target_pose": { + "name": "target_pose", + "description": "The target pose where the QR code should be placed.", + "modelname": None, # No specific model for this pose + } + } +} + +SCAN_OBJECT = { + "task_name": "scan_object", + "task_description": "Use one arm to pick the scanner and use the other arm to pick the object, and use the scanner to scan the object.\ + Move the scanner and object to the gripper. Get object target pose and place the object. Move the scanner to align with the object.\ + Note: The object target pose is dependent on the arm used to grasp the object.\ + The scanner should be placed at a distance of 0.05 meters from the functional point of the object.\ + You should not open the gripper after placing the object and scanner.", + "current_code": """ + class gpt_scan_object(scan_object): + def play_once(self): + pass + """, + "actor_list": { + "self.scanner": { + "name": "scanner", + "description": "The scanner that needs to be used.", + "modelname": "024_scanner", + }, + "self.object": { + "name": "object", + "description": "The object that needs to be scanned.", + "modelname": "112_tea-box", # The object can be any object that is specified in the task + }, + "self.left_object_target_pose": { + "name": "left_object_target_pose", + "description": "The target pose for the object when grasped with the left arm.", + "modelname": None, # No specific model for this pose + }, + "self.right_object_target_pose": { + "name": "right_object_target_pose", + "description": "The target pose for the object when grasped with the right arm.", + "modelname": None, # No specific model for this pose + }, + } +} + +STACK_BOWLS_THREE = { + "task_name": "stack_bowls_three", + "task_description": "Stack the three bowls on top of each other.\ + Move bowl 1 to the target pose, then move bowl 2 above bowl 1, and finally move bowl 3 above bowl 2.\ + Note: The target pose of bowl 2 is at 5 cm above bowl 1, and the target pose of bowl 3 is at 5 cm above bowl 2.\ + All target pose is np.ndarray([x, y, z]), so you should concatenate the quaternion later.", + "current_code": """ + class gpt_stack_bowls_three(stack_bowls_three): + def play_once(self): + pass + """, + "actor_list": { + "self.bowl1": { + "name": "bowl1", + "description": "The first bowl that should be placed at the bottom, you can get bowl1's position by using self.bowl1.get_pose().p.", + "modelname": "002_bowl", + }, + "self.bowl2": { + "name": "bowl2", + "description": "The second bowl that should be placed above the first bowl, you can get bowl1's position by using self.bowl1.get_pose().p, you can get the target pose of bowl 2 by adding 5 cm to the z-axis of bowl 1's target pose.", + "modelname": "002_bowl", + }, + "self.bowl3": { + "name": "bowl3", + "description": "The third bowl that should be placed above the second bowl, you can get bowl2's position by using self.bowl2.get_pose().p, you can get the target pose of bowl 3 by adding 5 cm to the z-axis of bowl 2's target pose.", + "modelname": "002_bowl", + }, + "self.bowl1_target_pose": { + "name": "bowl1_target_pose", + "description": "The target pose for the first bowl. It's a numpy.ndarray([x, y, z]) that should use .tolist() to be concatenated with the quaternion later.", + "modelname": None, # No specific model for this pose + }, + "self.quat_of_target_pose": { + "name": "quat_of_target_pose", + "description": "The quaternion of the target pose for the bowls, To be concatenated with the target pose.", + "modelname": None, # No specific model for this pose + }, + } +} + +STACK_BOWLS_TWO = { + "task_name": "stack_bowls_two", + "task_description": "Stack the two bowls on top of each other.\ + Move bowl 1 to the target pose, then move bowl 2 above bowl 1.\ + Note: The target pose of bowl 2 is at 5 cm above bowl 1.\ + All target pose is np.ndarray([x, y, z]), so you should concatenate the quaternion later.", + "current_code": """ + class gpt_stack_bowls_two(stack_bowls_two): + def play_once(self): + pass + """, + "actor_list": { + "self.bowl1": { + "name": "bowl1", + "description": "The first bowl that should be placed at the bottom, you can get bowl1's position by using self.bowl1.get_pose().p.", + "modelname": "002_bowl", + }, + "self.bowl2": { + "name": "bowl2", + "description": "The second bowl that should be placed above the first bowl, you can get bowl1's position by using self.bowl1.get_pose().p, you can get the target pose of bowl 2 by adding 5 cm to the z-axis of bowl 1's target pose.", + "modelname": "002_bowl", + }, + "self.bowl1_target_pose": { + "name": "bowl1_target_pose", + "description": "The target pose for the first bowl. It's a numpy.ndarray([x, y, z]) that should use .tolist() to be concatenated with the quaternion later.", + "modelname": None, # No specific model for this pose + }, + "self.quat_of_target_pose": { + "name": "quat_of_target_pose", + "description": "The quaternion of the target pose for the bowls, To be concatenated with the target pose.", + "modelname": None, # No specific model for this pose + }, + } +} + +#Note: You would better grasp the seal from top down direction. + +STAMP_SEAL = { + "task_name": "stamp_seal", + "task_description": "Use one arm to pick the stamp and place it on the target block.\ + Grasp the seal with specified arm. Place the seal on the target block.\ + Note: Don't set pre_dis_axis to fp, because the pre_dis_axis is not used in this task.", + "current_code": """ + class gpt_stamp_seal(stamp_seal): + def play_once(self): + pass + """, + "actor_list": { + "self.seal": { + "name": "seal", + "description": "The seal that needs to be placed on the target block.", + "modelname": "100_seal", + }, + "self.target_pose": { + "name": "target_pose", + "description": "The target pose where the seal should be placed on the target block.", + "modelname": None, # No specific model for this pose + } + } +} + + + +SHAKE_BOTTLE_HORIZONTALLY = { + "task_name": "shake_bottle_horizontally", + "task_description": "Shake the bottle horizontally with proper arm.\ + Grasp the bottle with specified arm. Shake the bottle horizontally by moving the arm left and right.", + "current_code": """ + class gpt_shake_bottle_horizontally(shake_bottle_horizontally): + def play_once(self): + pass + """, + "actor_list": { + "self.bottle": { + "name": "bottle", + "description": "The bottle that needs to be shaken horizontally.", + "modelname": "001_bottle", + } + } +} + + +SHAKE_BOTTLE = { + "task_name": "shake_bottle", + "task_description": "Shake the bottle with proper arm.\ + Grasp the bottle with specified arm. Shake the bottle by moving the arm up and down.", + "current_code": """ + class gpt_shake_bottle(shake_bottle): + def play_once(self): + pass + """, + "actor_list": { + "self.bottle": { + "name": "bottle", + "description": "The bottle that needs to be shaken.", + "modelname": "001_bottle", + } + } +} + + +PUT_BOTTLES_DUSTBIN = { + "task_name": "put_bottles_dustbin", + "task_description": "Use one arm to pick the bottle and put it into the dustbin.\ + Grasp the bottle with specified arm. Place the bottle into the dustbin. \ + Note: Don't set pre_dis_axis to fp, because the pre_dis_axis is not used in this task.", + "current_code": """ + class gpt_put_bottles_dustbin(put_bottles_dustbin): + def play_once(self): + pass + """, + "actor_list": { + "self.bottles[0]": { + "name": "bottle", + "description": "The first bottle that needs to be placed in the dustbin.", + "modelname": "114_bottle", + }, + "self.bottles[1]": { + "name": "bottle", + "description": "The second bottle that needs to be placed in the dustbin.", + "modelname": "114_bottle", + }, + "self.bottles[2]": { + "name": "bottle", + "description": "The third bottle that needs to be placed in the dustbin.", + "modelname": "114_bottle", + }, + "self.dustbin": { + "name": "dustbin", + "description": "The dustbin where the bottles need to be placed.", + "modelname": "011_dustbin", + }, + } +} + +TURN_SWITCH = { + "task_name": "turn_switch", + "task_description": "Use one arm to click the switch.\ + Close the gripper before clicking the switch. Then move the arm to the switch and click it.\ + Note: You can use grasp_actor() to simulate the action of clicking the switch.", + "current_code": """ + class gpt_turn_switch(turn_switch): + def play_once(self): + pass + """, + "actor_list": { + "self.switch": { + "name": "switch", + "description": "The switch that needs to be turned on or off.", + "modelname": "056_switch", + } + } +} + +OPEN_LAPTOP = { + "task_name": "open_laptop", + "task_description": "Open the laptop with one proper arm.\ + Grasp the laptop with specified arm. Open the laptop by moving the arm up.", + "current_code": """ + class gpt_open_laptop(open_laptop): + def play_once(self): + pass + """, + "actor_list": { + "self.laptop": { + "name": "laptop", + "description": "The laptop that needs to be opened.", + "modelname": "015_laptop", + } + } +} + +OPEN_MICROWAVE = { + "task_name": "open_microwave", + "task_description": "Open the microwave with one proper arm.\ + Grasp the handle of the microwave with specified arm. Pull the handle to open the microwave", + "current_code": """ + class gpt_open_microwave(open_microwave): + def play_once(self): + pass + """, + "actor_list": { + "self.microwave": { + "name": "microwave", + "description": "The microwave that needs to be opened.", + "modelname": "044_microwave", + } + } +} + +PUT_OBJECT_CABINET = { + "task_name": "put_object_cabinet", + "task_description": "Use one arm to open the cabinet, and use another arm to pick the object and put it into the cabinet.\ + Grasp the cabinet handle with one arm and open the cabinet. Grasp the object with the other arm and place it into the cabinet.", + "current_code": """ + class gpt_put_object_cabinet(put_object_cabinet): + def play_once(self): + pass + """, + "actor_list": { + "self.object": { + "name": "object", + "description": "The object that needs to be placed in the cabinet.", + "modelname": None, # Replace with actual model name + }, + "self.cabinet": { + "name": "cabinet", + "description": "The cabinet where the object needs to be placed.", + "modelname": "036_cabinet", + }, + } +} + +TURN_SWITCH = { + "task_name": "turn_switch", + "task_description": "Use one arm to click the switch.\ + Close the gripper before clicking the switch. Then move the arm to the switch and click it.\ + Note: You can use grasp_actor() to simulate the action of clicking the switch.", + "current_code": """ + class gpt_turn_switch(turn_switch): + def play_once(self): + pass + """, + "actor_list": { + "self.switch": { + "name": "switch", + "description": "The switch that needs to be turned on or off.", + "modelname": "056_switch", + } + } +} + + + +def get_all_tasks(): + return { + key: value + for key, value in globals().items() + if key.isupper() and isinstance(value, dict) and value # value非空dict + } + diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/test_gen_code.py b/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/test_gen_code.py new file mode 100644 index 00000000..6c278846 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/code_gen/test_gen_code.py @@ -0,0 +1,317 @@ +import sys + +sys.path.append("./") + +import sapien.core as sapien +from collections import OrderedDict +import pdb +from envs import * +import yaml +import importlib +import json +import traceback +import os +import time +import inspect + +current_file_path = os.path.abspath(__file__) +parent_directory = os.path.dirname(current_file_path) + +SCRIPT_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "script") +CONFIGS_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "task_config") +OBJECTS_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "assets/objects") + + +def enrich_actors(actor_list): + """ + Enrich the actor list by adding 'functional_points' and 'contact_points' + from the corresponding model metadata file, and remove the 'modelname' field + to make it suitable for prompting. + + Args: + actor_list (dict): Dictionary of actors with metadata. + + Returns: + dict: Enriched actor list with additional keys and without 'modelname'. + """ + enriched_actor_list = {} + + for actor_key, actor_info in actor_list.items(): + enriched_actor = actor_info.copy() + model_name = actor_info.get("modelname") + + if model_name is not None and model_name != "None": + points_info_path = os.path.join(OBJECTS_PATH, model_name, "points_info.json") + + if os.path.exists(points_info_path): + try: + with open(points_info_path, 'r') as f: + points_info = json.load(f) + + if "functional_points" in points_info: + enriched_actor["functional_points"] = points_info["functional_points"] + + if "contact_points" in points_info: + contact_points = points_info["contact_points"] + valid_contact_points = any( + point.get("id") is not None and ( + not isinstance(point.get("id"), list) or len(point.get("id")) > 0 + ) for point in contact_points + ) + enriched_actor["contact_points"] = contact_points if valid_contact_points else None + else: + enriched_actor["contact_points"] = None + + except Exception as e: + print(f"Error reading points_info.json for {model_name}: {e}") + print(traceback.format_exc()) + else: + print(f"Warning: File not found: {points_info_path}") + else: + print("modelname is None or invalid, skipping enrichment.") + + if "modelname" in enriched_actor: + del enriched_actor["modelname"] + + enriched_actor_list[actor_key] = enriched_actor + + return enriched_actor_list + + +def class_decorator_gen(task_name): + """ + Dynamically import and instantiate the task implementation from the code_gen module. + + Args: + task_name (str): Name of the task. + + Returns: + object: Instance of the task class. + """ + envs_module = importlib.import_module(f"envs_gen.gpt_{task_name}") + importlib.reload(envs_module) + try: + env_class = getattr(envs_module, f"gpt_{task_name}") + return env_class() + except: + raise SystemExit("No such task") + + +def class_decorator_env(task_name): + """ + Dynamically import and instantiate the task environment from the envs module. + + Args: + task_name (str): Name of the task. + + Returns: + object: Instance of the task class. + """ + envs_module = importlib.import_module(f"envs.{task_name}") + importlib.reload(envs_module) + try: + env_class = getattr(envs_module, task_name) + return env_class() + except: + raise SystemExit("No such task") + + +def create_task_config(task_config_path, task_name): + """ + Create a new task config file from the template if it doesn't exist. + + Args: + task_config_path (str): Path to the target config file. + task_name (str): Name of the task. + """ + with open(os.path.join(SCRIPT_PATH, "_task_config_template.json"), "r") as file: + task_config_template = json.load(file) + + # Modify task_name + task_config_template["task_name"] = task_name + + # Convert field format + if isinstance(task_config_template.get("embodiment"), str): + task_config_template["embodiment"] = [task_config_template["embodiment"]] + + # Save as yml + with open(task_config_path, "w") as f: + yaml.dump(task_config_template, f, default_flow_style=False, sort_keys=False) + + +def get_embodiment_config(robot_file): + """ + Load embodiment configuration from the robot folder. + + Args: + robot_file (str): Path to the robot folder. + + Returns: + dict: Robot configuration. + """ + robot_config_file = os.path.join(robot_file, "config.yml") + with open(robot_config_file, "r", encoding="utf-8") as f: + return yaml.load(f.read(), Loader=yaml.FullLoader) + + +def setup_task_config(task_name): + """ + Load or create a task configuration and set up robot embodiments. + + Args: + task_name (str): Task name. + + Returns: + tuple: (Task instance, task configuration dictionary) + """ + task = class_decorator_gen(task_name) + task_config_path = f"./task_config/{task_name}.yml" + + if not os.path.isfile(task_config_path): + create_task_config(task_config_path, task_name) + print(f"Task config file is missing, please check {task_config_path}") + + with open(task_config_path, "r", encoding="utf-8") as f: + args = yaml.load(f.read(), Loader=yaml.FullLoader) + + args["domain_randomization"] = { + "random_background": False, + "cluttered_table": False, + "clean_background_rate": 0.0, + "random_head_camera_dis": 0, + "random_table_height": 0.0, + "random_light": False, + "crazy_random_light_rate": 0.0, + "random_embodiment": False, + } + + embodiment_type = args.get("embodiment") + embodiment_config_path = os.path.join("./task_config", "_embodiment_config.yml") + with open(embodiment_config_path, "r", encoding="utf-8") as f: + _embodiment_types = yaml.load(f.read(), Loader=yaml.FullLoader) + + def get_embodiment_file(embodiment_type): + robot_file = _embodiment_types[embodiment_type]["file_path"] + if robot_file is None: + raise Exception("No embodiment files") + return robot_file if os.path.isabs(robot_file) else os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", robot_file) + ) + + if len(embodiment_type) == 1: + args["left_robot_file"] = get_embodiment_file(embodiment_type[0]) + args["right_robot_file"] = get_embodiment_file(embodiment_type[0]) + args["dual_arm_embodied"] = True + elif len(embodiment_type) == 3: + args["left_robot_file"] = get_embodiment_file(embodiment_type[0]) + args["right_robot_file"] = get_embodiment_file(embodiment_type[1]) + args["embodiment_dis"] = embodiment_type[2] + args["dual_arm_embodied"] = False + else: + raise Exception("Embodiment items should be 1 or 3") + + args["left_embodiment_config"] = get_embodiment_config(args["left_robot_file"]) + args["right_embodiment_config"] = get_embodiment_config(args["right_robot_file"]) + + args["embodiment_name"] = ( + str(embodiment_type[0]) if len(embodiment_type) == 1 + else str(embodiment_type[0]) + "+" + str(embodiment_type[1]) + ) + + args["need_plan"] = True + args["save_path"] = "./data/test" + + return task, args + + +def run(TASK_ENV, args, check_num=10): + """ + Run the task in simulation to evaluate success rate. + + Args: + TASK_ENV (object): Task environment instance. + args (dict): Task configuration. + check_num (int): Number of trials to run. + + Returns: + tuple: (success rate, most common error message, error count, run records) + """ + epid, suc_num, fail_num = 0, 0, 0 + + error_list = [ + "The code can not run", "The left arm failed to grasp the object", "The right arm failed to grasp the object", + "The target position of the object is incorrect.", "Plan execution failed", + "Unknown error occurred during execution" + ] + error_num = [0, 0, 0, 0, 0, 0] + run_records = [] + + print(f"\033[34mTask name: {args['task_name']}\033[0m") + print("\033[93m" + "[Start Testing Task Success Rate]" + "\033[0m") + + print("\n\033[92m=== play_once source code ===\033[0m") + play_once_method = TASK_ENV.__class__.play_once + print(inspect.getsource(play_once_method)) + print("\033[92m=== End ===\033[0m\n") + + for epid in range(check_num): + error_id = None + try: + TASK_ENV.setup_demo(now_ep_num=suc_num, seed=epid, **args) + TASK_ENV.play_once() + + if TASK_ENV.plan_success and TASK_ENV.check_success(): + print(f"simulate data episode {suc_num} success! (seed = {epid})") + suc_num += 1 + run_records.append("success!") + else: + if not TASK_ENV.plan_success: + if hasattr(TASK_ENV, 'left_plan_success') and not TASK_ENV.lefft_plan_success: + error_id = 1 + run_records.append(error_list[1]) + elif hasattr(TASK_ENV, 'right_plan_success') and not TASK_ENV.right_plan_success: + error_id = 2 + run_records.append(error_list[2]) + else: + error_id = 4 + run_records.append(error_list[4]) + else: + error_id = 3 + run_records.append(error_list[3]) + + print(f"simulate data episode {suc_num} fail! (seed = {epid})") + fail_num += 1 + + TASK_ENV.close() + if args.get("render_freq"): + TASK_ENV.viewer.close() + + except Exception as e: + error_id = 0 + error_list[0] = str(traceback.format_exc()) + run_records.append(f"Error: {e}") + print("-------------") + print(f"simulate data episode {suc_num} fail! (seed = {epid})") + print("Error:", traceback.format_exc()) + print("-------------") + fail_num += 1 + TASK_ENV.close() + if args.get("render_freq"): + TASK_ENV.viewer.close() + time.sleep(2) + + if error_id is not None: + error_num[error_id] += 1 + + if len(run_records) != check_num: + print(f"Warning: number of records ({len(run_records)}) does not match number of trials ({check_num})") + + max_error_index = error_num.index(max(error_num)) if sum(error_num) > 0 else 5 + max_error_count = error_num[max_error_index] + + print(f'\nComplete test, success rate: {suc_num}/{check_num}') + print(f'Error message: {error_list}') + print(f'Run records: {run_records}') + print(f'error_num: {error_num}') + + return suc_num / check_num, error_list[max_error_index], max_error_count, run_records diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/collect_data.sh b/operator-images/builtin/holobrain-vendor/RoboTwin/collect_data.sh new file mode 100644 index 00000000..0c6c0ff2 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/collect_data.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +task_name=${1} +task_config=${2} +gpu_id=${3} + +./script/.update_path.sh > /dev/null 2>&1 + +export CUDA_VISIBLE_DEVICES=${gpu_id} + +PYTHONWARNINGS=ignore::UserWarning \ +python script/collect_data.py $task_name $task_config +rm -rf data/${task_name}/${task_config}/.cache diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/data/process_stuck.py b/operator-images/builtin/holobrain-vendor/RoboTwin/data/process_stuck.py new file mode 100644 index 00000000..8eaba033 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/data/process_stuck.py @@ -0,0 +1,34 @@ +import os +import argparse + +ap = argparse.ArgumentParser() +ap.add_argument("task_name") +ap.add_argument("task_config") +ap.add_argument("index", type=int, help="Index of the episode to delete") +args = ap.parse_args() + +target_path = os.path.join("data", args.task_name, args.task_config) +txt_path = os.path.join(target_path, "seed.txt") +dir_path = os.path.join(target_path, "data") +replace_id = args.index + +with open(txt_path, "r") as f: + nums = list(map(int, f.read().split())) + +final_id = len(nums) - 1 + +last_seed = nums[-1] + +nums[replace_id] = last_seed +nums.pop() + +with open(txt_path, "w") as f: + f.write(" ".join(map(str, nums))) + +target_file = os.path.join(dir_path, f"episode{replace_id}.pkl") +if os.path.exists(target_file): + os.remove(target_file) + +last_file = os.path.join(dir_path, f"episode{final_id}.pkl") +if os.path.exists(last_file): + os.rename(last_file, target_file) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/_generate_object_prompt.txt b/operator-images/builtin/holobrain-vendor/RoboTwin/description/_generate_object_prompt.txt new file mode 100644 index 00000000..0d0f287a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/_generate_object_prompt.txt @@ -0,0 +1,32 @@ +Look at the image and generate 15 descriptions of the main object shown. Each description should: + +1. THIS IS THE MOST IMPORTANT:Use natural spoken/VERY oral language style(IMPORTANT! just think of what people will normally refer to it as) +2. Avoid articles (a, an, the) and commas +3. Vary in length. Maximum 6 words, THE SHORTER AND MORE PRECISE, THE MERRIER. AVOID USING DIFFICULT WORDS. BE EASY TO UNDERSTAND AND ORAL. +4. Progress from simple to detailed descriptions +5. Use different word syntax structures across descriptions +6. Be primarily noun-focused phrases +7. EMPHASIZE physical properties essential for manipulation: + - First explicitly speak out the different features with oral words + - COLOR (be specific about shades and patterns) + - SHAPE (describe geometric form precisely, and handles/bars) + - SIZE (relative dimensions and scale) + - TEXTURE/MATERIAL (when visible) + - SUBPART's INFO + +8. Each be distinctly different in wording and detail level +9. REALLY IMPORTANT!!avoid using abstract words like 'object' 'device' 'container'. +10. THIS IS ALSO THE MOST IMPORTANT:confirm that ANY person can know what you are talking about, after only reading ONE DESCRIPTION +11. If the object contains multiple parts, describe it using structural phrases including but not limited to "X with Y" (e.g., "bottle with yellow lid"). If no multi-component feature is shown, neglect this requirement. +12. Do not use question marks or interrogative sentences +The user will first tell you the object's GROUND TRUTH SHORT NAME (maybe with unnecessary prefixs and surfixs, which you can neglect), which will help you recognize it. +IMPORTANT:Make sure the object's text name (without id prefixs or _ symbols) are present in ALL!!!!! of the descriptions you generate + +Format your response as required by the response_format. + +Example (if the image showed a Coca-Cola can): +1. red can +2. Coca-Cola can +3. small metallic red soda can +4. red can with white Coca-Cola label +5. palm-sized beverage can \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/_generate_task_prompt.txt b/operator-images/builtin/holobrain-vendor/RoboTwin/description/_generate_task_prompt.txt new file mode 100644 index 00000000..54b86966 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/_generate_task_prompt.txt @@ -0,0 +1,12 @@ +# TASK: ABSTRACT ROBOTIC TASK DESCRIPTIONS + +## OBJECTIVE +1. Provide a concise description of the task flow. +2. Avoid including very detailed specifics (e.g., exact coordinates), + but make sure each '<' '>' wraped highlight point is clearly mentioned in each of the instruction +3. Use natural, action-oriented verbs like "grab", "slide", "set", "stick", "drop", "place", etc., instead of technical jargon. +4. Vary sentence structures (e.g., questions, commands, requests) and maintain a natural, conversational tone. +5. Generate a given number of alternative descriptions based on the input. +6. Avoid question marks and unnecessary words. +7. Avoid adding unnecessary ADJECTIVES or adverbs at the end of sentences!!!!! +8. Clearly or implicitly include all steps of the task in each instruction. \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/_generate_task_prompt_schema.txt b/operator-images/builtin/holobrain-vendor/RoboTwin/description/_generate_task_prompt_schema.txt new file mode 100644 index 00000000..f1975e3b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/_generate_task_prompt_schema.txt @@ -0,0 +1,6 @@ +# schema requirements + +## OBJECTIVE +1. Use placeholders in the format {X} for objects, where X is defined in a schema. +2. Ensure all object placeholders ({A-Z}) are included in every instruction, but REFERENCE TO ARMS, INCLUDING arm placeholders ({a-z}) MUST be omitted in 50% of the instructions. +3. Make sure instructions flow naturally when placeholders ({A-Za-z}) are replaced with actual objects or arm notations. \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/gen_episode_instructions.sh b/operator-images/builtin/holobrain-vendor/RoboTwin/description/gen_episode_instructions.sh new file mode 100644 index 00000000..11f4da95 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/gen_episode_instructions.sh @@ -0,0 +1,5 @@ +task_name=${1} +setting=${2} +max_num=${3} + +python utils/generate_episode_instructions.py $task_name $setting $max_num diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/gen_object_descriptions.sh b/operator-images/builtin/holobrain-vendor/RoboTwin/description/gen_object_descriptions.sh new file mode 100644 index 00000000..490a7587 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/gen_object_descriptions.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# 获取传入的参数 +object_name=${1} +object_id=${2} + +# 检查是否提供了足够的参数 +if [ -z "$object_name" ]; then + echo "Error: object_name is required." + echo "Usage: $0 [object_id]" + exit 1 +fi + +# 检查 object_id 是否为空 +if [ -z "$object_id" ]; then + # 如果 object_id 为空,传递一个空字符串 + python utils/generate_object_description.py "$object_name" +else + # 如果 object_id 不为空,正常传递 + python utils/generate_object_description.py "$object_name" --index "$object_id" +fi \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/gen_task_instruction_templates.sh b/operator-images/builtin/holobrain-vendor/RoboTwin/description/gen_task_instruction_templates.sh new file mode 100644 index 00000000..8cf9e660 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/gen_task_instruction_templates.sh @@ -0,0 +1,4 @@ +task_name=${1} +instruction_num=${2} + +python utils/generate_task_description.py $task_name $instruction_num \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base0.json new file mode 100644 index 00000000..4fa666b8 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "red bottle", + "plastic bottle", + "yellow text bottle", + "bottle with red cap", + "handheld plastic bottle", + "red bottle with screw cap", + "medium-size plastic bottle", + "red bottle with narrow neck", + "smooth red cylindrical bottle", + "red bottle with curved grooves", + "red bottle with white bottom text", + "plastic bottle with bright red color" + ], + "unseen": [ + "shiny red bottle", + "bottle with white and yellow label", + "bottle with yellow Chinese characters" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base1.json new file mode 100644 index 00000000..ef8cafe7 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "yellow bottle", + "plastic bottle", + "bottle for liquid", + "medium-sized bottle", + "bottle with screw cap", + "smooth plastic bottle", + "handheld curvy bottle", + "yellow bottle with blue label", + "yellow bottle with dark label", + "medium plastic bottle with cap", + "plastic bottle with curved sides", + "yellow bottle for liquid storage" + ], + "unseen": [ + "curvy yellow plastic bottle", + "smooth bottle with blue text", + "yellow bottle with white cap" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base10.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base10.json new file mode 100644 index 00000000..cd14fd56 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base10.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "brown bottle", + "plastic bottle", + "bottle with green cap", + "bottle with flat sides and logo", + "medium-sized brown plastic bottle", + "medium bottle with glossy surface", + "smooth plastic bottle with white cap", + "smooth brown bottle with green label", + "dark brown bottle with a squared body", + "rectangular brown body with white top", + "rectangular-shaped bottle with round cap", + "brown bottle with rounded rectangular shape" + ], + "unseen": [ + "liquid container dark brown plastic", + "bottle with green label and white lid", + "medium-sized plastic bottle with capped top" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base11.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base11.json new file mode 100644 index 00000000..92d00361 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base11.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "yellow bottle", + "plastic bottle", + "bottle with white cap", + "bottle with green label", + "smooth bottle with rounded edges", + "medium rectangular yellow bottle", + "yellow bottle with a green design", + "bottle with a smooth plastic body", + "bottle with green and white labels", + "yellow container with rounded edges", + "yellow plastic bottle with white top", + "rectangular bottle with cylindrical neck" + ], + "unseen": [ + "yellow rectangular bottle", + "hand-sized yellow plastic bottle", + "yellow bottle for liquid storage" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base12.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base12.json new file mode 100644 index 00000000..eaf5d0cd --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base12.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "green bottle", + "plastic bottle", + "bottle with yellow cap", + "handheld plastic bottle", + "medium bottle with green body", + "green bottle with white label", + "smooth green and yellow bottle", + "bottle with glossy plastic finish", + "plastic bottle with colored label", + "green bottle with black text label", + "cylindrical bottle with rounded cap", + "cylindrical liquid bottle with yellow top" + ], + "unseen": [ + "medium-sized drink bottle", + "bottle with green base and black prints", + "yellow-capped bottle with smooth surface" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base13.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base13.json new file mode 100644 index 00000000..471c630b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base13.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "long thin bottle", + "Coca-Cola bottle", + "red-capped bottle", + "plastic drink bottle", + "narrow-necked bottle", + "hand-sized soda bottle", + "medium dark brown bottle", + "smooth bottle with red lid", + "dark drink bottle with logo", + "brown bottle with red label", + "bottle with ridges on bottom", + "bottle with white neck and red top" + ], + "unseen": [ + "brown bottle", + "plastic soda bottle", + "bottle with red screw cap" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base14.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base14.json new file mode 100644 index 00000000..ce82acf1 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base14.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "blue bottle", + "plastic bottle", + "handheld bottle", + "cylindrical bottle", + "bottle for liquids", + "medium-sized bottle", + "bottle with blue cap", + "white and blue bottle", + "smooth plastic bottle", + "blue bottle with label", + "medium bottle for beverages", + "white bottle with blue accents" + ], + "unseen": [ + "bottle with ridged neck", + "blue-capped plastic bottle", + "bottle with curved neck design" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base15.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base15.json new file mode 100644 index 00000000..3b3b4131 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base15.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "smooth brown bottle", + "bottle with white cap", + "medium plastic bottle", + "plastic bottle for liquids", + "bottle with dark brown color", + "plastic bottle with printed label", + "medium bottle with smooth texture", + "cylindrical bottle with narrow neck", + "rounded bottle with printed details", + "handheld brown bottle with white lid", + "bottle with brown body and white top", + "white-capped brown cylindrical bottle" + ], + "unseen": [ + "brown bottle", + "lightweight plastic brown bottle", + "brown cylindrical bottle with label" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base16.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base16.json new file mode 100644 index 00000000..268ce07e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base16.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "plastic bottle", + "medium green bottle", + "smooth green bottle", + "handheld green bottle", + "bottle with narrow top", + "bottle with rounded neck", + "medium-sized green bottle", + "smooth green plastic bottle", + "bottle with ridges near base", + "green bottle with indented base", + "bottle with white printed label", + "medium green bottle with capped neck" + ], + "unseen": [ + "green bottle", + "green bottle with white label", + "green plastic bottle with ridged bottom" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base17.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base17.json new file mode 100644 index 00000000..2ae94205 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base17.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "white bottle", + "smooth bottle", + "plastic bottle", + "medium-sized bottle", + "white cylindrical bottle", + "white bottle for liquids", + "bottle with tapering neck", + "white bottle with red text", + "bottle with orange screw cap", + "white bottle with orange lid", + "handheld white bottle with cap", + "cylindrical bottle with red writing" + ], + "unseen": [ + "bottle with orange cap", + "bottle with glossy surface", + "plastic bottle with printed label" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base18.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base18.json new file mode 100644 index 00000000..354b8f1b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base18.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "milk-bottle shape design", + "smooth cylindrical bottle", + "medium-sized plastic bottle", + "medium cylinder-shaped bottle", + "gray bottle with slight shine", + "bottle with cute ghost drawing", + "plastic bottle with ghost print", + "hand-sized bottle with screw cap", + "bottle shaped like small cylinder", + "light gray colored container bottle", + "rounded body bottle made of plastic", + "smooth surface bottle with gray shade" + ], + "unseen": [ + "light gray bottle", + "bottle with rounded cap top", + "gray bottle for holding drinks" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base19.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base19.json new file mode 100644 index 00000000..2b591125 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base19.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "blue bottle", + "plastic bottle", + "medium water bottle", + "portable bottle for liquids", + "light blue cylindrical bottle", + "plastic bottle with black top", + "rounded bottle with black lid", + "bottle with black textured cap", + "light blue bottle with strap-loop", + "light blue bottle with smooth body", + "blue water bottle with textured lid", + "cylindrical bottle with rounded edges" + ], + "unseen": [ + "smooth blue water bottle", + "bottle with attached loop on cap", + "cylindrical plastic bottle for carrying water" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base2.json new file mode 100644 index 00000000..527067c8 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "black bottle", + "yellow capped drink bottle", + "medium bottle with yellow cap", + "smooth black bottle with ridges", + "beverage bottle with black body", + "yellow-capped cylindrical bottle", + "bottle with blue and yellow print", + "plastic bottle with colorful label", + "black plastic bottle with Pepsi logo", + "drink bottle with rounded yellow lid", + "black cylindrical bottle with ridges", + "medium black bottle with printed label" + ], + "unseen": [ + "bottle with ridges near base", + "plastic bottle with blue label", + "hand-sized bottle with ridged bottom" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base20.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base20.json new file mode 100644 index 00000000..850e70fb --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base20.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "white bottle", + "smooth plastic bottle", + "bottle with white screw-top lid", + "bottle with brown carrying loop", + "yellow and blue patterned bottle", + "white bottle with rounded bottom", + "medium bottle with screw-top cap", + "bottle decorated in floral prints", + "plastic bottle with floral designs", + "plastic bottle with yellow flowers", + "cylindrical bottle with handle loop", + "white bottle with blue leaf designs" + ], + "unseen": [ + "medium-sized bottle for liquids", + "bottle with attached loop and lid", + "white bottle with colorful patterns" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base21.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base21.json new file mode 100644 index 00000000..3f12036e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base21.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "smooth bottle", + "plastic bottle", + "rounded bottle", + "bottle for liquids", + "glossy plastic bottle", + "palm-sized pink bottle", + "bottle with yellow accent", + "bottle with glossy finish", + "bright pink cylindrical bottle", + "small pink bottle with black top", + "bright pink bottle with black cap", + "rounded plastic bottle with color patterns" + ], + "unseen": [ + "pink bottle", + "small bottle", + "bottle with white text" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base22.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base22.json new file mode 100644 index 00000000..ef29f892 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base22.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "green bottle", + "light green drink bottle", + "bottle with green screw cap", + "handheld light green bottle", + "curved plastic bottle with cap", + "green bottle with textured cap", + "smooth bottle with printed label", + "green bottle with glossy surface", + "plastic bottle for holding liquid", + "green cylindrical plastic container", + "green bottle with grip-friendly cap", + "light green bottle with printed text" + ], + "unseen": [ + "medium green plastic bottle", + "plastic bottle with tapered body", + "medium-sized green cylindrical bottle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base3.json new file mode 100644 index 00000000..bbbddd9f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "yellow bottle", + "plastic bottle", + "handheld bottle", + "rectangular bottle", + "yellow body bottle", + "bottle for liquids", + "bottle with white cap", + "smooth yellow plastic bottle", + "yellow bottle with green label", + "yellow bottle with flower design", + "bottle with green and white label", + "yellow bottle with cylindrical cap" + ], + "unseen": [ + "medium bottle", + "bottle with printed label", + "bottle with rounded edges" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base4.json new file mode 100644 index 00000000..c4394fe6 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "orange bottle", + "plastic bottle", + "smooth orange bottle", + "curved orange soda bottle", + "bottle for holding drinks", + "grooved orange drink bottle", + "light and medium-sized bottle", + "orange drink bottle with label", + "medium bottle with colorful label", + "ridged orange bottle with grooves", + "cylindrical orange plastic bottle", + "round-bottom bottle with white cap" + ], + "unseen": [ + "bottle with white cap", + "bottle with ridged orange grip", + "white capped orange plastic bottle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base5.json new file mode 100644 index 00000000..c11d59f4 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "orange bottle", + "plastic bottle", + "yellow label bottle", + "medium orange bottle", + "smooth plastic bottle", + "bottle with orange cap", + "medium bottle with orange lid", + "white and orange plastic bottle", + "bottle with ribbed orange bottom", + "orange and white bottle with cap", + "drinking bottle with yellow label", + "cylindrical orange and white bottle" + ], + "unseen": [ + "bottle with curved body", + "bottle with yellow sticker label", + "curvy orange bottle with white section" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base6.json new file mode 100644 index 00000000..f551a62b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "orange bottle", + "plastic bottle", + "bottle for liquids", + "smooth plastic bottle", + "bottle with yellow cap", + "bottle with narrow neck", + "bottle with printed label", + "orange cylindrical bottle", + "medium-sized orange bottle", + "yellow-capped plastic bottle", + "plastic bottle with sealed yellow cap", + "cylindrical orange bottle with yellow top" + ], + "unseen": [ + "handheld orange bottle", + "label-covered orange bottle", + "orange bottle with smooth texture" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base7.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base7.json new file mode 100644 index 00000000..85b60714 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base7.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "orange bottle", + "bottle with yellow cap", + "smooth orange cylindrical bottle", + "orange bottle with bold labeling", + "liquid container with orange body", + "yellow-capped bottle for beverages", + "yellow-capped orange plastic bottle", + "handheld medium-sized liquid bottle", + "small bottle with bright yellow lid", + "rounded bottom orange plastic bottle", + "rounded orange bottle with yellow top", + "cylindrical orange bottle with smooth surface" + ], + "unseen": [ + "medium plastic bottle", + "medium bottle for liquids", + "plastic orange bottle with rounded base" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base8.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base8.json new file mode 100644 index 00000000..c4dc12a7 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base8.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "brown bottle", + "plastic bottle", + "white-capped bottle", + "bottle holding liquid", + "bottle with black label", + "hand-sized plastic bottle", + "small curved plastic bottle", + "bottle with white screw cap", + "curved bottle with white top", + "small bottle with decorative design", + "bottle with brown and white patterns", + "smooth brown and white patterned bottle" + ], + "unseen": [ + "glossy brown bottle", + "brown bottle with glossy swirls", + "cylindrical bottle with curved sides" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base9.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base9.json new file mode 100644 index 00000000..e86e934e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/001_bottle/base9.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "brown bottle", + "green-capped bottle", + "bottle with green cap", + "small brown plastic bottle", + "small drink bottle with logo", + "hand-sized bottle with grooves", + "plastic bottle with ribbed pattern", + "cylindrical bottle with white label", + "smooth plastic bottle for beverages", + "brown bottle with green screw-on cap", + "bottle with tapered cylindrical shape", + "brown bottle with white printed label" + ], + "unseen": [ + "plastic Starbucks bottle", + "bottle with coffee-colored body", + "bottle with Starbucks logo on label" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/002_bowl/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/002_bowl/base1.json new file mode 100644 index 00000000..7b188cc8 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/002_bowl/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bowl", + "seen": [ + "white bowl", + "round white bowl", + "simple serving bowl", + "bowl for holding items", + "bowl with black interior", + "bowl with deep black hollow", + "white bowl with curved shape", + "smooth bowl with curved edges", + "medium bowl for food or items", + "white bowl with smooth texture", + "ceramic bowl with glossy surface", + "round ceramic bowl with black inner" + ], + "unseen": [ + "palm-sized glossy bowl", + "black and white round bowl", + "medium-sized white serving bowl" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/002_bowl/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/002_bowl/base2.json new file mode 100644 index 00000000..703e6696 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/002_bowl/base2.json @@ -0,0 +1,21 @@ +{ + "raw_description": "bowl", + "seen": [ + "gray bowl", + "ceramic bowl", + "medium ceramic bowl", + "medium bowl for food", + "light gray round bowl", + "light gray ceramic bowl", + "ceramic bowl light gray", + "medium-sized smooth bowl", + "round bowl with open top", + "bowl with smooth surface", + "open-top round light gray bowl" + ], + "unseen": [ + "simple round bowl", + "smooth light gray bowl", + "light gray smooth round bowl" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/002_bowl/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/002_bowl/base3.json new file mode 100644 index 00000000..a9ad28ba --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/002_bowl/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bowl", + "seen": [ + "blue bowl", + "ceramic bowl", + "bowl with shiny texture", + "bowl with brown outer rim", + "small bowl with brown edge", + "curved bowl light blue shade", + "glazed bowl with curved sides", + "rounded bowl with light blue color", + "medium-sized bowl for holding food", + "ceramic bowl with earthy brown trim", + "dark inside bowl light blue outside", + "blue bowl with slightly rounded base" + ], + "unseen": [ + "smooth bowl with glossy finish", + "small smooth brown-rimmed bowl", + "light blue bowl with darkened rim" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/002_bowl/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/002_bowl/base4.json new file mode 100644 index 00000000..4d101666 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/002_bowl/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bowl", + "seen": [ + "round bowl", + "bowl with yellow dots", + "small smooth white bowl", + "round bowl with open top", + "yellow-spotted ceramic bowl", + "medium bowl with glossy finish", + "simple round bowl with yellow spots", + "smooth ceramic bowl for serving food", + "white bowl with circular yellow spots", + "hand-sized bowl with a smooth texture", + "bowl with white base and yellow accents", + "glossy white bowl featuring yellow circles" + ], + "unseen": [ + "white bowl", + "yellow-patterned bowl with shiny surface", + "dish-shaped white bowl with yellow designs" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/002_bowl/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/002_bowl/base5.json new file mode 100644 index 00000000..54af688e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/002_bowl/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bowl", + "seen": [ + "bowl", + "gray bowl", + "ceramic bowl", + "smooth gray bowl", + "medium-sized bowl", + "smooth light gray bowl", + "light gray ceramic bowl", + "rounded bowl with open top", + "light gray medium-sized bowl", + "gray bowl with rounded edges", + "gray bowl with polished surface", + "gray ceramic bowl with open top" + ], + "unseen": [ + "rounded bowl", + "bowl for holding items", + "curved bowl with smooth texture" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/002_bowl/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/002_bowl/base6.json new file mode 100644 index 00000000..1e21e189 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/002_bowl/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bowl", + "seen": [ + "gray bowl", + "smooth bowl", + "ceramic bowl", + "medium-sized bowl", + "concave gray bowl", + "light gray food bowl", + "light gray round bowl", + "concave bowl gray ceramic", + "round bowl polished surface", + "smooth light gray round bowl", + "medium ceramic bowl for food", + "round bowl with smooth finish" + ], + "unseen": [ + "bowl for holding liquid", + "ceramic bowl medium in size", + "polished ceramic bowl light gray" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/002_bowl/base7.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/002_bowl/base7.json new file mode 100644 index 00000000..63d8615e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/002_bowl/base7.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bowl", + "seen": [ + "smooth green bowl", + "round plastic bowl", + "food-holding green bowl", + "rounded light green bowl", + "light green bowl for food", + "two-hand-sized green bowl", + "deep bowl with round shape", + "medium-sized light green bowl", + "bowl with deep smooth interior", + "bowl with slightly ridged sides", + "smooth-textured light green bowl", + "plastic bowl with concave interior" + ], + "unseen": [ + "light green bowl", + "bowl with ridged exterior", + "green round bowl for liquids" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/003_plate/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/003_plate/base0.json new file mode 100644 index 00000000..3de87ecd --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/003_plate/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "plate", + "seen": [ + "dish plate", + "food plate", + "round plate", + "smooth ceramic plate", + "medium plate for food", + "medium round flat plate", + "flat round greenish plate", + "medium round ceramic plate", + "smooth greenish table plate", + "greenish-gray flat dinner plate", + "ceramic plate with smooth surface", + "flat ceramic plate with smooth texture" + ], + "unseen": [ + "gray plate", + "gray plate for dining", + "smooth round plate for serving" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/004_fluted-block/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/004_fluted-block/base0.json new file mode 100644 index 00000000..76c7ce3f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/004_fluted-block/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "fluted block", + "seen": [ + "dark gray fluted block", + "irregular fluted block", + "block with matte finish", + "angled block in dark gray", + "block with angled surfaces", + "smooth-sided dark gray block", + "solid dark gray fluted block", + "solid block with angled faces", + "gray block with corner indent", + "block with smooth gray texture", + "irregular dark gray angled block", + "hard gray block with inner shape" + ], + "unseen": [ + "medium fluted block", + "structural fluted block", + "dark gray block with inner corner" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/004_fluted-block/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/004_fluted-block/base1.json new file mode 100644 index 00000000..ce01235b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/004_fluted-block/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "fluted block", + "seen": [ + "medium fluted block", + "dark gray trapezoid", + "solid dark gray block", + "block with curved side", + "fluted block with curve", + "curved gray fluted block", + "medium gray block with curve", + "block with trapezoidal sides", + "smooth gray trapezoidal block", + "solid trapezoidal fluted block", + "smooth medium trapezoidal block", + "trapezoidal block with curved groove" + ], + "unseen": [ + "dark gray fluted block", + "gray block with indentation", + "matte gray trapezoidal block" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/005_french-fries/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/005_french-fries/base0.json new file mode 100644 index 00000000..c7e932c5 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/005_french-fries/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "french fries", + "seen": [ + "red fries packet", + "McDonald's fries bag", + "small red fries packet", + "bright red fries holder", + "thin crispy potato fries", + "red box with golden sticks", + "rectangular red fries packet", + "golden fries in red packaging", + "red container with yellow fries", + "crispy fries in hand-sized pouch", + "golden fries in smooth red pouch", + "McDonald's logo on red fries box" + ], + "unseen": [ + "hand-sized fries container", + "golden fries in red holder", + "golden crispy fries inside red box" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/005_french-fries/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/005_french-fries/base1.json new file mode 100644 index 00000000..c79e5576 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/005_french-fries/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "French fries", + "seen": [ + "Red fries box", + "Small red box fries inside", + "French fries in red packaging", + "Palm-sized red fries container", + "Bright red box with fries inside", + "Light yellow fries in red holder", + "Golden fries sticks in red carton", + "Open-top red box filled with fries", + "Potato fries packaged in red carton", + "Rectangular red carton holding fries", + "Handheld fries pack bright red color", + "Golden fries crispy red package container" + ], + "unseen": [ + "Golden fries in red carton box", + "Crispy potato fries with red box", + "Red carton box fries crispy golden" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/005_french-fries/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/005_french-fries/base2.json new file mode 100644 index 00000000..5183cbac --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/005_french-fries/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "french fries", + "seen": [ + "yellow fries", + "crispy french fries", + "medium fries serving", + "hand-sized fries box", + "fries in red container", + "golden thin potato fries", + "crispy thin fries in box", + "golden fries with red box", + "rectangular fries container", + "yellow crispy fries with box", + "fried potato in multicolor box", + "red container filled with fries" + ], + "unseen": [ + "potato fries sticks", + "multicolor fries box", + "fries with colorful packaging" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/005_french-fries/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/005_french-fries/base3.json new file mode 100644 index 00000000..6b4091a7 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/005_french-fries/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "french fries", + "seen": [ + "golden fries", + "long thin fries", + "crispy french fries", + "red pouch with fries", + "medium-sized fries pouch", + "palm-sized fries container", + "french fries in red carton", + "red carton with golden fries", + "smooth red pouch crispy fries", + "golden fries in open red pouch", + "golden yellow fried potato sticks", + "crispy fried potato sticks in pouch" + ], + "unseen": [ + "hand-size fries bag", + "fried potato snack in red holder", + "cardboard pouch holding crispy fries" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/006_hamburg/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/006_hamburg/base0.json new file mode 100644 index 00000000..4c11a82e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/006_hamburg/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "hamburg", + "seen": [ + "round hamburg", + "smooth hamburg", + "bread bun hamburg", + "light brown hamburg", + "medium-sized hamburg", + "brownish round hamburg", + "hamburg with uneven top", + "medium, squishy hamburg", + "soft brown bread hamburg", + "round, soft bread hamburg", + "hamburg with white patches", + "hamburg with patterned brown surface" + ], + "unseen": [ + "palm-sized hamburg", + "hamburg with white sides", + "hamburg with light brown top" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/006_hamburg/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/006_hamburg/base1.json new file mode 100644 index 00000000..4ea17c0e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/006_hamburg/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "hamburg", + "seen": [ + "orange hamburg", + "small rounded hamburg", + "hamburg with yellow mark", + "irregular shaped hamburg", + "hand-sized orange hamburg", + "decorative rounded hamburg", + "smooth orange and white hamburg", + "orange hamburg with white underside", + "white part with orange edges hamburg", + "rounded hamburg with painted details", + "yellow marked white and orange hamburg", + "orange-white hamburg with yellow highlight" + ], + "unseen": [ + "hamburg with soft texture", + "hamburg with curved surfaces", + "hamburg with white and orange" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/006_hamburg/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/006_hamburg/base2.json new file mode 100644 index 00000000..21f39473 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/006_hamburg/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "hamburg", + "seen": [ + "white hamburg", + "hamburg with crumpled edges", + "palm-sized hamburg packaging", + "smooth white hamburg packaging", + "yellow and brown patterned hamburg", + "white hamburg with creased wrapper", + "hamburg with printed illustrations", + "hamburg with yellow and brown marks", + "white hamburg with yellow and brown stains", + "smooth white hamburg wrap with yellow spots", + "rectangular hamburg wrap with printed design", + "wrapped hamburg with white plastic-like cover" + ], + "unseen": [ + "rounded rectangular hamburg wrap", + "hamburg with uneven white surface", + "small yellow-marked hamburg wrapper" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/006_hamburg/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/006_hamburg/base3.json new file mode 100644 index 00000000..9353bfdb --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/006_hamburg/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "hamburg", + "seen": [ + "hand-sized hamburg", + "easy-to-hold hamburg", + "small soft hamburg in paper", + "hamburg with printed wrapper", + "hamburg wrapped in white paper", + "golden brown and white hamburg", + "soft hamburg with smooth wrapper", + "wrapped hamburg with green design", + "hamburg with smooth paper wrapping", + "hamburg with rounded rectangular shape", + "rectangular hamburg with rounded edges", + "white wrapper enclosing golden hamburg" + ], + "unseen": [ + "golden brown hamburg", + "small rectangular hamburg", + "golden brown food within wrapper" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/006_hamburg/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/006_hamburg/base4.json new file mode 100644 index 00000000..24bf8da2 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/006_hamburg/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "hamburg", + "seen": [ + "yellow hamburg box", + "yellow box for hamburg", + "rectangular hamburg box", + "hamburg box with foldable top", + "foldable hamburg cardboard box", + "smooth yellow hamburg packaging", + "medium-sized yellow hamburg package", + "bright yellow hamburg box with logo", + "medium-sized box with hamburg design", + "yellow hamburg box with white accents", + "white and yellow printed hamburg holder", + "white lid with yellow bottom hamburg box" + ], + "unseen": [ + "cardboard hamburg container", + "hamburg pack with printed details", + "rectangular cardboard box for hamburg" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/006_hamburg/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/006_hamburg/base5.json new file mode 100644 index 00000000..f5cc099b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/006_hamburg/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "hamburg", + "seen": [ + "small hamburg box", + "smooth hamburg box", + "hamburg box with lid", + "green top hamburg box", + "rectangular hamburg box", + "yellow bottom hamburg box", + "easy-to-carry hamburg box", + "box with green hamburg lid", + "hamburg box with rounded edges", + "yellow and green hamburg holder", + "green and yellow hamburg container", + "rectangular hamburg container with smooth finish" + ], + "unseen": [ + "compact hamburg box", + "stackable hamburg box", + "cardboard hamburg box" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/007_shoe-box/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/007_shoe-box/base0.json new file mode 100644 index 00000000..b70fd667 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/007_shoe-box/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "shoe-box", + "seen": [ + "orange shoe-box", + "bright orange shoe-box", + "smooth bright orange box", + "orange cardboard shoe-box", + "rectangular orange shoe-box", + "smooth orange box for shoes", + "Nike-branded orange shoe-box", + "orange Nike shoe-box with logo", + "shoe-box with Nike printed logo", + "medium-sized box with Nike swoosh", + "rectangular shoe-box with orange lid", + "medium orange box with smooth surface" + ], + "unseen": [ + "medium cardboard shoe-box", + "orange box for keeping shoes", + "rectangular Nike box for shoes" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/008_tray/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/008_tray/base0.json new file mode 100644 index 00000000..a1883797 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/008_tray/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tray", + "seen": [ + "beige tray", + "tray for holding items", + "smooth rectangular tray", + "light beige smooth tray", + "medium rectangular flat tray", + "flat tray with rounded corners", + "plastic tray for carrying items", + "smooth light beige plastic tray", + "medium beige tray with flat base", + "tray with smooth plastic surface", + "rectangular beige tray for objects", + "tray with rounded rectangular shape" + ], + "unseen": [ + "rectangular beige tray", + "medium beige plastic tray", + "beige tray with soft edges" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/008_tray/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/008_tray/base1.json new file mode 100644 index 00000000..b9066a1c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/008_tray/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tray", + "seen": [ + "orange tray", + "rectangular tray", + "smooth plastic tray", + "medium bright orange tray", + "medium-sized plastic tray", + "bright orange rectangular tray", + "plastic tray for holding items", + "bright orange tray for serving", + "plastic tray with shiny texture", + "orange tray with smooth surface", + "smooth glossy orange medium tray", + "bright orange tray with glossy finish" + ], + "unseen": [ + "rectangular tray with rounded edges", + "rectangular bright orange serving tray", + "medium-sized tray with rounded corners" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/008_tray/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/008_tray/base2.json new file mode 100644 index 00000000..794d21df --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/008_tray/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tray", + "seen": [ + "medium plastic tray", + "rectangular green tray", + "tray for serving items", + "dark green serving tray", + "tray for holding objects", + "tray with rounded corners", + "smooth texture green tray", + "flat green rectangular tray", + "solid green palm-sized tray", + "single-piece dark green tray", + "plastic tray with curved edges", + "dark green rectangular plastic tray" + ], + "unseen": [ + "green tray", + "smooth green tray", + "medium green tray for carrying" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/008_tray/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/008_tray/base3.json new file mode 100644 index 00000000..e66e1883 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/008_tray/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tray", + "seen": [ + "brown tray", + "rectangular tray", + "smooth plastic tray", + "brown tray with rim", + "medium dark brown tray", + "tray for holding things", + "rectangular plastic tray", + "medium-sized dark brown tray", + "dark rectangular serving tray", + "flat tray with smooth surface", + "tray with slightly raised edges", + "flat brown tray with raised edges" + ], + "unseen": [ + "medium flat tray", + "tray for carrying items", + "flat brown serving tray" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/009_kettle/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/009_kettle/base0.json new file mode 100644 index 00000000..8385b32d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/009_kettle/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "kettle", + "seen": [ + "green kettle", + "medium kettle with spout", + "kettle with smooth finish", + "smooth bright green kettle", + "bright green painted kettle", + "kettle with butterfly design", + "green kettle with black handle", + "medium green kettle with spout", + "round kettle with black handle", + "metal kettle with butterfly logo", + "green kettle with butterfly print", + "painted kettle with butterfly logo" + ], + "unseen": [ + "round green kettle", + "bright green kettle", + "green water-boiling kettle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/009_kettle/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/009_kettle/base1.json new file mode 100644 index 00000000..d43bd183 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/009_kettle/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "kettle", + "seen": [ + "gray kettle", + "smooth gray kettle", + "curved spout kettle", + "kettle with round lid", + "black handle on kettle", + "kettle with black handle", + "kettle for boiling water", + "metal kettle for pouring water", + "rounded kettle with black grip", + "gray kettle with smooth surface", + "rounded kettle with curved spout", + "medium kettle with curved tube spout" + ], + "unseen": [ + "metal kettle with lid", + "medium-sized metal kettle", + "gray kettle with wide base" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/009_kettle/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/009_kettle/base2.json new file mode 100644 index 00000000..8d7d6daa --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/009_kettle/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "kettle", + "seen": [ + "white kettle", + "round white kettle", + "medium round kettle", + "kettle with red handle", + "red-handled white kettle", + "ribbed red kettle handle", + "smooth kettle with red handle", + "kettle with smooth metal body", + "metal kettle with curved spout", + "white kettle with curved spout", + "metal kettle with plastic handle", + "round kettle with dark red details" + ], + "unseen": [ + "kettle for boiling water", + "medium-sized white kettle", + "white kettle with ribbed handle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/011_dustbin/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/011_dustbin/base0.json new file mode 100644 index 00000000..a6d8adb0 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/011_dustbin/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "dustbin", + "seen": [ + "trash bin", + "black dustbin", + "plastic dustbin", + "black garbage bin", + "smooth black dustbin", + "dustbin with lid and bag", + "black bin with white liner", + "dustbin with white inner bag", + "black and white trash holder", + "rectangular black trash holder", + "medium black garbage container", + "medium-sized black plastic dustbin" + ], + "unseen": [ + "smooth surface dustbin", + "simple black garbage bin", + "curved rectangular trash can" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/012_plant-pot/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/012_plant-pot/base0.json new file mode 100644 index 00000000..5426070f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/012_plant-pot/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "plant-pot", + "seen": [ + "red green plant-pot", + "square red plant-pot", + "small glossy plant-pot", + "handheld glossy plant-pot", + "watermelon-themed plant-pot", + "plant-pot with green leaves", + "ceramic watermelon plant-pot", + "rounded watermelon plant-pot", + "green rimmed square plant-pot", + "green and red ceramic plant-pot", + "plant-pot with watermelon print", + "decorative plant-pot for small plants" + ], + "unseen": [ + "red and green ceramic pot", + "red pot with black seeds design", + "ceramic pot shaped like watermelon" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/012_plant-pot/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/012_plant-pot/base1.json new file mode 100644 index 00000000..a5c1864b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/012_plant-pot/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "plant pot", + "seen": [ + "plant pot", + "orange plant pot", + "smooth orange plant pot", + "two-hand-size plant pot", + "medium plant pot for soil", + "rectangular dark orange pot", + "angled rectangular plant pot", + "dark orange pot for gardening", + "plastic pot for holding plants", + "plastic orange rectangular planter", + "plastic rectangular orange plant holder", + "medium rectangular pot with smooth sides" + ], + "unseen": [ + "dark orange soil pot", + "smooth-textured orange pot", + "rectangular plastic plant pot" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/012_plant-pot/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/012_plant-pot/base2.json new file mode 100644 index 00000000..e8e9ff0f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/012_plant-pot/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "plant-pot", + "seen": [ + "black plant-pot", + "purple-flowered plant-pot", + "plant-pot with green leaves", + "purple flowers in plant-pot", + "medium pot with green plants", + "medium rectangular plant-pot", + "pot with purple flowering plants", + "black pot with spreading greenery", + "smooth black rectangular plant-pot", + "plant-pot holding soil and flowers", + "black rectangular pot with flowers", + "rectangular plant-pot with open top" + ], + "unseen": [ + "black base with leafy plants", + "plastic pot containing plants", + "decorative plant-pot with leafy plants" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/012_plant-pot/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/012_plant-pot/base3.json new file mode 100644 index 00000000..b8cc92c5 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/012_plant-pot/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "plant pot", + "seen": [ + "black plant pot", + "small plastic plant pot", + "plant pot with green leaves", + "dark black pot with greenery", + "plant pot small with dark hue", + "plant pot smooth black surface", + "green plant in black plant pot", + "black pot with curved inward edges", + "compact black pot holding fake plant", + "rounded edge black pot with greenery", + "smooth black pot with green plant top", + "plastic black pot with synthetic plant" + ], + "unseen": [ + "hand-sized plant pot", + "rounded rectangular black plant pot", + "small pot with decorative artificial leaves" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/012_plant-pot/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/012_plant-pot/base4.json new file mode 100644 index 00000000..b925a7d1 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/012_plant-pot/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "plant-pot", + "seen": [ + "purple plant-pot", + "smooth plant-pot", + "floral plant-pot", + "ceramic plant-pot", + "decorative plant-pot", + "ceramic pot for plants", + "medium purple plant-pot", + "painted purple plant-pot", + "oval-shaped purple plant-pot", + "rounded rectangular plant-pot", + "purple pot with rounded edges", + "medium-sized ceramic plant-pot" + ], + "unseen": [ + "floral-patterned plant-pot", + "plant-pot with flower designs", + "purple pot with smooth surface" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/013_dumbbell-rack/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/013_dumbbell-rack/base0.json new file mode 100644 index 00000000..f6e5060d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/013_dumbbell-rack/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "dumbbell rack", + "seen": [ + "gym dumbbell rack", + "rack for dumbbells", + "metal dumbbell rack", + "silver dumbbell rack", + "two-tier dumbbell rack", + "rack with dumbbell slots", + "rectangular dumbbell rack", + "smooth metal dumbbell rack", + "two-layer silver metal rack", + "silver rack for gym dumbbells", + "blue and silver dumbbell holder", + "dumbbell rack with wide shelves" + ], + "unseen": [ + "medium dumbbell rack", + "compact dumbbell rack for weights", + "blue-framed dumbbell rack with shelves" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/013_dumbbell-rack/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/013_dumbbell-rack/base1.json new file mode 100644 index 00000000..119cc020 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/013_dumbbell-rack/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "dumbbell rack", + "seen": [ + "dumbbell rack", + "gym dumbbell rack", + "metal dumbbell rack", + "two-tier dumbbell rack", + "sturdy dumbbell holder", + "dumbbell rack with slots", + "black slots on metal rack", + "rack for storing dumbbells", + "angled white dumbbell stand", + "dumbbell rack for gym weights", + "medium-sized gym dumbbell rack", + "smooth white frame dumbbell rack" + ], + "unseen": [ + "white dumbbell storage rack", + "angled dumbbell storage stand", + "white rack with black holders" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/013_dumbbell-rack/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/013_dumbbell-rack/base2.json new file mode 100644 index 00000000..1dd4de5e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/013_dumbbell-rack/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "dumbbell rack", + "seen": [ + "rack for dumbbells", + "weight holder rack", + "black dumbbell rack", + "metal dumbbell rack", + "gym dumbbell holder", + "medium dumbbell rack", + "black rack for weights", + "sturdy metal dumbbell rack", + "gray and black dumbbell rack", + "compact dumbbell storage rack", + "dumbbell rack with smooth bars", + "rectangular dumbbell holder frame" + ], + "unseen": [ + "metal rack with gray supports", + "rack for organizing gym dumbbells", + "dumbbell rack with horizontal bars" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/013_dumbbell-rack/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/013_dumbbell-rack/base3.json new file mode 100644 index 00000000..531afa4d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/013_dumbbell-rack/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "dumbbell rack", + "seen": [ + "red dumbbell rack", + "rack with dumbbells", + "black dumbbells on rack", + "sturdy rack for dumbbells", + "rack for holding dumbbells", + "red rack for gym dumbbells", + "black and red dumbbell rack", + "red rack with multiple slots", + "dumbbell rack with curved slots", + "metal and plastic dumbbell rack", + "rectangular-shaped dumbbell rack", + "dumbbell rack with black weights" + ], + "unseen": [ + "medium dumbbell rack", + "compact dumbbell rack", + "red curved slots dumbbell rack" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/014_bookcase/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/014_bookcase/base0.json new file mode 100644 index 00000000..1036664e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/014_bookcase/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bookcase", + "seen": [ + "light brown bookcase", + "rectangular wooden bookcase", + "bookcase with vertical slots", + "medium wooden book organizer", + "light brown shelves for books", + "smooth brown bookcase for books", + "bookcase with varied book sizes", + "bookcase with flat wooden shelves", + "wooden bookcase with smooth texture", + "grain-textured light brown bookcase", + "rectangular bookcase with side panels", + "rectangular bookcase with smooth grain" + ], + "unseen": [ + "medium-sized bookshelf", + "bookcase holding multiple books", + "bookcase with vertical wooden supports" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/014_bookcase/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/014_bookcase/base1.json new file mode 100644 index 00000000..9db2139a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/014_bookcase/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bookcase", + "seen": [ + "bookcase made from wood material", + "medium shelf for books and items", + "bookcase with books and plant pot", + "wooden bookcase with open sections", + "medium rectangular shelf with slots", + "rectangular shelf for books and decor", + "orange-brown bookcase with smooth wood", + "wooden storage unit with vertical panels", + "compact orange-brown bookcase with items", + "bookshelf having distinct circular cutouts", + "rectangular bookcase having dividers inside", + "orange-brown storage piece with smooth finish" + ], + "unseen": [ + "orange-brown bookcase", + "bookcase with circular hole design", + "orange bookshelf featuring grain texture" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/014_bookcase/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/014_bookcase/base2.json new file mode 100644 index 00000000..e75e4a53 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/014_bookcase/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bookcase", + "seen": [ + "wooden bookcase", + "light brown bookcase", + "compact tabletop bookcase", + "medium-sized wooden bookcase", + "organized shelves in bookcase", + "bookcase with multicolored books", + "books neatly stacked in bookcase", + "rectangular bookcase with shelves", + "small bookcase with multiple sections", + "bookcase holding books and small items", + "bookcase with visible books and shelves", + "light brown rectangular bookcase with compartments" + ], + "unseen": [ + "bookcase with smooth finish", + "clean and smooth wooden bookcase", + "functional wooden bookcase for storage" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/014_bookcase/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/014_bookcase/base3.json new file mode 100644 index 00000000..75479be8 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/014_bookcase/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bookcase", + "seen": [ + "light blue bookcase", + "bookcase with open slots", + "rectangular blue bookcase", + "shiny light blue bookcase", + "medium-sized smooth bookcase", + "bookcase with handle cut-outs", + "bookcase with flat panel dividers", + "blue bookcase with glossy surface", + "bookcase with rectangular openings", + "bookcase with storage compartments", + "plastic bookcase with smooth finish", + "bookcase featuring ergonomic handles" + ], + "unseen": [ + "boxy plastic bookcase", + "compact light blue bookcase", + "bookcase divided into sections" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base0.json new file mode 100644 index 00000000..a381616f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "laptop", + "seen": [ + "medium-sized laptop", + "silver keyboard laptop", + "black and silver laptop", + "portable folding laptop", + "laptop with silver keyboard", + "matte laptop with smooth screen", + "compact black and silver computer", + "Toshiba laptop with folding design", + "black laptop with reflective screen", + "rectangular laptop with Toshiba logo", + "device with flat keyboard and screen", + "folding laptop with screen and keyboard" + ], + "unseen": [ + "laptop computer", + "portable Toshiba laptop", + "computer with black screen" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base1.json new file mode 100644 index 00000000..1c66ce3c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "laptop", + "seen": [ + "slim laptop", + "portable slim laptop", + "compact silver laptop", + "lightweight silver laptop", + "medium-sized silver laptop", + "laptop with black keyboard", + "laptop with smooth surface", + "laptop with metallic finish", + "metal laptop with black keys", + "silver laptop with dark keys", + "rectangular laptop with screen", + "laptop with foldable screen design" + ], + "unseen": [ + "silver laptop", + "foldable laptop", + "folding laptop with black keyboard" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base10.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base10.json new file mode 100644 index 00000000..6aee735c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base10.json @@ -0,0 +1,22 @@ +{ + "raw_description": "laptop", + "seen": [ + "metal laptop", + "silver laptop", + "keyboard laptop", + "black screen laptop", + "wood surrounded laptop", + "smooth textured laptop", + "black and silver laptop", + "medium foldable silver laptop", + "laptop with black textured screen", + "silver laptop with foldable parts", + "silver laptop with black keyboard", + "laptop with wood panel background" + ], + "unseen": [ + "foldable laptop", + "medium laptop with silver body", + "laptop with glossy black display" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base2.json new file mode 100644 index 00000000..ff26827c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "laptop", + "seen": [ + "metallic laptop", + "blue-screen laptop", + "medium-sized laptop", + "laptop for computing", + "portable silver laptop", + "laptop with blue screen", + "laptop with silver base", + "laptop with glass screen", + "hinged laptop with display", + "laptop with smooth texture", + "rectangular laptop with hinge", + "laptop featuring smooth metal finish" + ], + "unseen": [ + "sleek laptop", + "silver laptop", + "compact silver laptop" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base3.json new file mode 100644 index 00000000..6c60f339 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "laptop", + "seen": [ + "black laptop", + "laptop with blue screen", + "portable folding laptop", + "rectangular black laptop", + "screen and keyboard laptop", + "laptop with rectangular base", + "laptop with smooth matte surface", + "laptop with smooth keyboard area", + "blue-screen laptop with flat keys", + "folding laptop with black exterior", + "sleek black laptop with blue monitor", + "black plastic laptop with blue display" + ], + "unseen": [ + "medium-sized black laptop", + "black laptop with shiny hinges", + "black laptop with compact design" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base4.json new file mode 100644 index 00000000..02607717 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "laptop", + "seen": [ + "black laptop", + "laptop with wide black lid", + "laptop with silver keyboard", + "black laptop with smooth body", + "laptop with shiny black surface", + "black and silver portable laptop", + "clamshell laptop with flat screen", + "laptop featuring built-in keyboard", + "rectangular laptop with folding hinge", + "rectangular laptop with smooth texture", + "laptop with rectangular display screen", + "laptop with silver buttons on keyboard" + ], + "unseen": [ + "medium laptop with black case", + "laptop with black cover and bottom", + "laptop with smooth edges and flat case" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base5.json new file mode 100644 index 00000000..631cd5ce --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "laptop", + "seen": [ + "black laptop", + "laptop with glossy screen", + "laptop for typing and computing", + "black laptop with visible hinge", + "laptop with flat black keyboard", + "black laptop with metallic edges", + "medium-sized black portable laptop", + "black laptop with shiny display panel", + "black laptop with Saturn image on screen", + "smooth laptop with Windows logo underneath", + "portable black laptop with textured casing", + "laptop featuring black base and hinged screen" + ], + "unseen": [ + "thin rectangular laptop", + "laptop with colorful screen", + "hinged laptop with keyboard and screen" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base6.json new file mode 100644 index 00000000..a17582ac --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "laptop", + "seen": [ + "orange laptop", + "foldable laptop", + "thin portable laptop", + "hinged screen laptop", + "laptop with blue screen", + "laptop with folding lid", + "blue screen portable laptop", + "blue screen foldable laptop", + "laptop with black hinge area", + "medium rectangular orange laptop", + "rectangular laptop with orange body", + "plastic orange laptop with metal parts" + ], + "unseen": [ + "flat orange laptop", + "laptop with smooth surfaces", + "compact laptop with orange exterior" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base7.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base7.json new file mode 100644 index 00000000..59892827 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base7.json @@ -0,0 +1,22 @@ +{ + "raw_description": "laptop", + "seen": [ + "black laptop", + "Toshiba laptop", + "foldable black laptop", + "medium black rectangular laptop", + "portable folding Toshiba laptop", + "black laptop with hinge mechanism", + "black laptop with red Toshiba logo", + "sleek black Toshiba computing laptop", + "black laptop with red text underneath", + "black laptop with smooth matte surface", + "Toshiba laptop with rectangular screen", + "folding laptop with screen and keyboard" + ], + "unseen": [ + "black laptop fitting laptop bags", + "medium-sized laptop for portable use", + "black laptop slightly rounded at edges" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base8.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base8.json new file mode 100644 index 00000000..b09fa13b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base8.json @@ -0,0 +1,22 @@ +{ + "raw_description": "laptop", + "seen": [ + "portable laptop", + "blue-black laptop", + "compact black laptop", + "blue base black lid laptop", + "foldable laptop with screen", + "medium-sized laptop for work", + "laptop for browsing and computing", + "blue bottom foldable black laptop", + "black laptop with blue bottom panel", + "black-and-blue laptop with keyboard", + "rectangular laptop with foldable hinge", + "black laptop with sharp rectangular edges" + ], + "unseen": [ + "black laptop", + "smooth black plastic laptop", + "laptop with display and keyboard" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base9.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base9.json new file mode 100644 index 00000000..3af7d12a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/015_laptop/base9.json @@ -0,0 +1,22 @@ +{ + "raw_description": "laptop", + "seen": [ + "black laptop", + "lightweight laptop", + "sleek laptop design", + "compact black laptop", + "black metallic laptop", + "black laptop with red accents", + "laptop with red logo underside", + "laptop with screen and keyboard", + "hinged laptop with screen and base", + "laptop featuring textured exterior", + "medium-sized laptop with black body", + "foldable laptop with patterned keyboard" + ], + "unseen": [ + "portable laptop", + "rectangular laptop with smooth finish", + "laptop with patterned keyboard surface" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/017_calculator/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/017_calculator/base0.json new file mode 100644 index 00000000..32c3c1e1 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/017_calculator/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "calculator", + "seen": [ + "black calculator", + "plastic calculator", + "calculator for quick math", + "calculator with button grid", + "calculator with slanted top", + "calculator with blue buttons", + "small rectangular calculator", + "calculator with smooth surface", + "black calculator with slanted body", + "calculator with black glossy finish", + "calculator with black and blue colors", + "compact calculator for handling numbers" + ], + "unseen": [ + "handheld calculator", + "calculator with display screen", + "calculator with white and blue keys" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/017_calculator/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/017_calculator/base1.json new file mode 100644 index 00000000..92139e04 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/017_calculator/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "calculator", + "seen": [ + "black calculator", + "plastic calculator", + "handheld calculator", + "black rectangular calculator", + "calculator with green screen", + "calculator with slanted top edge", + "small calculator with numeric keypad", + "calculator with smooth black surface", + "calculator with black and gray display", + "compact calculator for math operations", + "calculator with slanted rectangular shape", + "calculator with number and function buttons" + ], + "unseen": [ + "calculator with embossed keys", + "calculator with white and red keys", + "calculator with visible button layout" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/017_calculator/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/017_calculator/base2.json new file mode 100644 index 00000000..c49b67dd --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/017_calculator/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "calculator", + "seen": [ + "black and white calculator", + "calculator with angled base", + "compact handheld calculator", + "small rectangular calculator", + "black calculator with white keys", + "calculator with buttons and screen", + "calculator with smooth plastic body", + "white and black calculator for math", + "plastic calculator with gray buttons", + "calculator with clear digital display", + "black base calculator with white front", + "rectangular calculator with raised buttons" + ], + "unseen": [ + "calculator with display at top", + "calculator showing numbers on screen", + "calculator with slanted black bottom" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/017_calculator/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/017_calculator/base3.json new file mode 100644 index 00000000..8203c734 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/017_calculator/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "calculator", + "seen": [ + "thin rectangular calculator", + "calculator with black buttons", + "calculator with dark green screen", + "palm-sized rectangular calculator", + "calculator with raised black keys", + "white and black number calculator", + "calculator with smooth plastic body", + "smooth white calculator with display", + "compact handheld calculator for math", + "calculator with rows of black buttons", + "black-button calculator with white body", + "basic calculation device with green screen" + ], + "unseen": [ + "white calculator", + "small white arithmetic calculator", + "white calculator with rectangular screen" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/017_calculator/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/017_calculator/base4.json new file mode 100644 index 00000000..08539ff9 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/017_calculator/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "calculator", + "seen": [ + "small plastic calculator", + "compact black calculator", + "calculator with angled sides", + "calculator with white buttons", + "rectangular handheld calculator", + "calculator with smooth black body", + "calculator for math and accounting", + "black calculator with display screen", + "calculator with raised round buttons", + "calculator with rows of white buttons", + "calculator with number and function keys", + "calculator with large rectangular display" + ], + "unseen": [ + "black calculator", + "calculator with black plastic casing", + "black calculator with smooth texture" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/017_calculator/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/017_calculator/base5.json new file mode 100644 index 00000000..d35c67fc --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/017_calculator/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "calculator", + "seen": [ + "white calculator", + "basic calculator", + "calculator for math tasks", + "calculator with plastic body", + "calculator with black display", + "calculator with rectangular shape", + "white calculator with blue accents", + "calculator with blue and white keys", + "calculator screen with glossy finish", + "calculator with smooth textured buttons", + "rectangular calculator with rounded edges", + "calculator with numeric and function buttons" + ], + "unseen": [ + "palm-sized calculator", + "small handheld calculator", + "black calculator with white buttons" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/018_microphone/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/018_microphone/base0.json new file mode 100644 index 00000000..b90367b9 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/018_microphone/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "microphone", + "seen": [ + "teal microphone", + "handheld teal microphone", + "rounded white tip microphone", + "microphone with slider switch", + "textured white microphone head", + "white and teal sound microphone", + "sound microphone with teal body", + "compact teal and white microphone", + "microphone with white rounded head", + "plastic teal microphone with slider", + "long microphone with smooth teal grip", + "white bottom microphone with textured tip" + ], + "unseen": [ + "teal microphone with white bottom", + "switchable microphone with teal body", + "long teal microphone with white grooves" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/018_microphone/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/018_microphone/base1.json new file mode 100644 index 00000000..9321789e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/018_microphone/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "microphone", + "seen": [ + "black microphone", + "handheld microphone", + "handheld white body microphone", + "white microphone with mesh head", + "microphone with silver mesh head", + "audio microphone silver mesh head", + "black plastic and metal microphone", + "black and white rectangular microphone", + "microphone with black vertical cylinder", + "silver mesh microphone with black cylinder", + "microphone with rectangular base and round top", + "white rectangular microphone with spherical head" + ], + "unseen": [ + "round-headed white microphone", + "mesh head microphone with black top", + "microphone with smooth plastic body" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/018_microphone/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/018_microphone/base4.json new file mode 100644 index 00000000..d2c913aa --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/018_microphone/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "microphone", + "seen": [ + "dark blue microphone", + "round head microphone", + "small audio microphone", + "gray and dark blue microphone", + "microphone for recording sound", + "microphone with round foam head", + "microphone with smooth gray stem", + "microphone covered with foam tip", + "microphone with dark blue padding", + "gray stem microphone with foam top", + "audio microphone with soft covering", + "medium microphone with two-tone color" + ], + "unseen": [ + "handheld microphone", + "compact microphone with two parts", + "microphone with cylindrical handle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/018_microphone/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/018_microphone/base5.json new file mode 100644 index 00000000..4a0e1281 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/018_microphone/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "microphone", + "seen": [ + "handheld microphone", + "metal head microphone", + "compact size microphone", + "black and white microphone", + "mesh-patterned microphone head", + "microphone for voice recording", + "smooth plastic microphone handle", + "microphone with rounded mesh head", + "microphone with cylindrical handle", + "microphone with textured metal head", + "white microphone handle black accents", + "microphone with black tip and white body" + ], + "unseen": [ + "microphone used for audio input", + "lightweight handheld microphone", + "black and white plastic microphone" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/019_coaster/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/019_coaster/base0.json new file mode 100644 index 00000000..99f63769 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/019_coaster/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "coaster", + "seen": [ + "round coaster", + "wooden coaster", + "grainy texture coaster", + "flat round wooden coaster", + "coaster with circular shape", + "coaster made of smooth wood", + "smooth round palm-sized coaster", + "light gray round grainy coaster", + "small coaster with grain texture", + "flat surface gray wooden coaster", + "round coaster with light streaks", + "light gray streaked wooden coaster" + ], + "unseen": [ + "light gray coaster", + "disc-shaped light gray coaster", + "small drink-protecting wooden coaster" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/020_hammer/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/020_hammer/base0.json new file mode 100644 index 00000000..cdc95da7 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/020_hammer/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "hammer", + "seen": [ + "silver hammer", + "hammer for nails", + "nail-driving hammer", + "grippy handle hammer", + "medium-sized metal hammer", + "silver curved hammer head", + "hammer with claw-shaped end", + "hammer with two-tone handle", + "plastic handle metal hammer", + "handheld medium claw hammer", + "yellow and black hammer grip", + "black and yellow hammer grip" + ], + "unseen": [ + "hammer with black handle", + "silver hammer head and claw", + "hammer with claw and smooth head" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base0.json new file mode 100644 index 00000000..1baa0a69 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "cup", + "seen": [ + "rounded base blue cup", + "light blue plastic cup", + "plastic cup for drinks", + "cup for holding liquids", + "blue rounded-bottom cup", + "smooth blue drinking cup", + "cup with light blue color", + "cylindrical light blue cup", + "medium blue cylindrical cup", + "smooth blue cup for liquids", + "medium-sized plastic blue cup", + "cup with smooth plastic surface" + ], + "unseen": [ + "blue cup", + "small smooth cup", + "handheld round blue cup" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base1.json new file mode 100644 index 00000000..0319fc63 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "cup", + "seen": [ + "white cup", + "small ceramic cup", + "blue and white cup", + "rounded ceramic cup", + "cup with blue flowers", + "cup shaped like a cylinder", + "white cup with rounded base", + "blue patterned drinking cup", + "small cup with floral design", + "small cup for drinking liquids", + "palm-sized cup with smooth surface", + "white ceramic cup with blue flowers" + ], + "unseen": [ + "glossy cup with floral patterns", + "ceramic drinking cup with floral design", + "ceramic cup featuring blue flower prints" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base10.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base10.json new file mode 100644 index 00000000..de09fda7 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base10.json @@ -0,0 +1,22 @@ +{ + "raw_description": "cup", + "seen": [ + "white cup", + "rounded white tea cup", + "cup with black patterns", + "cup with smooth surface", + "handle cup in palm-size", + "medium white ceramic cup", + "white glossy ceramic cup", + "medium-sized drinking cup", + "glossy white cup with designs", + "ceramic cup with curved handle", + "black patterned white coffee cup", + "cylindrical cup with decorated exterior" + ], + "unseen": [ + "cup with black abstract lines", + "cup with artistic black swirls", + "cylindrical cup with circular handle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base11.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base11.json new file mode 100644 index 00000000..557931ce --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base11.json @@ -0,0 +1,22 @@ +{ + "raw_description": "cup", + "seen": [ + "black cup", + "ceramic cup", + "medium-sized black cup", + "cup with curved handle", + "palm-sized ceramic cup", + "cup for holding liquids", + "curved handled black cup", + "black smooth texture cup", + "black cylinder-shaped cup", + "smooth shiny black drinking cup", + "medium black ceramic cup handle", + "cylinder-shaped cup black finish" + ], + "unseen": [ + "black cup shiny finish", + "black cup with single handle", + "drinking cup medium size black" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base12.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base12.json new file mode 100644 index 00000000..af869b5d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base12.json @@ -0,0 +1,22 @@ +{ + "raw_description": "cup", + "seen": [ + "black cup", + "ceramic cup", + "cylindrical cup", + "smooth black cup", + "black drinking cup", + "black cup with handle", + "black medium-sized cup", + "cup with rounded handle", + "barrel-shaped black cup", + "medium black ceramic cup", + "cup with smooth black body", + "shiny black cup with curved handle" + ], + "unseen": [ + "cup for liquids", + "black coffee cup", + "black cup for hot drinks" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base2.json new file mode 100644 index 00000000..94b81f0b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "cup", + "seen": [ + "brown cup", + "plastic cup", + "dark brown ribbed cup", + "cup with ribbed sides", + "medium-sized brown cup", + "cup with ridges for grip", + "brown cup smooth top edge", + "ribbed brown cylinder cup", + "brown plastic cup smooth top", + "drinking cup medium palm size", + "cup shaped like ribbed cylinder", + "dark ribbed plastic drinking cup" + ], + "unseen": [ + "ridged cylindrical cup", + "simple dark brown plastic cup", + "brown cylinder cup holds liquids" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base3.json new file mode 100644 index 00000000..c72fd42a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "cup", + "seen": [ + "plastic cup", + "dark green cup", + "cylindrical cup", + "medium-sized cup", + "simple drinking cup", + "cup with tapered sides", + "hand-sized plastic cup", + "smooth cylindrical cup", + "dark green plastic tapered cup", + "dark green smooth drinking cup", + "medium green cup with smooth sides", + "tapered cylindrical plastic green cup" + ], + "unseen": [ + "green cup", + "cup for liquids", + "smooth green cup" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base4.json new file mode 100644 index 00000000..1ab34944 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "cup", + "seen": [ + "white cup", + "rounded white cup", + "white drinking cup", + "cup with black base", + "ceramic-like white cup", + "hand-sized white bowl cup", + "cup with black circular base", + "white cup for serving drinks", + "bowl-shaped white ceramic cup", + "medium cup for holding liquids", + "round white cup with narrow base", + "white glossy cup with black foot" + ], + "unseen": [ + "black-bottomed cup", + "small round drinkable cup", + "cup with smooth glossy surface" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base5.json new file mode 100644 index 00000000..57d47d71 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "cup", + "seen": [ + "gray cup", + "metal cup", + "dark gray cylinder cup", + "cup with rough texture", + "cup for holding liquids", + "brown and gray metal cup", + "medium-sized beverage cup", + "hand-sized rough metal cup", + "cup with worn metal finish", + "simple dark gray drinking cup", + "gray cup with faded brown spots", + "cylindrical cup with grainy surface" + ], + "unseen": [ + "cup made of metal", + "cup with rounded edges", + "rusty-looking grayish metallic cup" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base6.json new file mode 100644 index 00000000..c79687e4 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "cup", + "seen": [ + "silver cup", + "metallic cup", + "silver cup for drinks", + "medium silver metal cup", + "cup with metallic finish", + "medium-sized silver holder", + "cup with curved metal handle", + "smooth cylindrical silver cup", + "metal cup with smooth texture", + "silver cup with hollow design", + "medium shiny silver cylinder cup", + "cylinder-shaped metal beverage cup" + ], + "unseen": [ + "shiny silver drinking cup", + "drinking cup made of metal", + "cup with curved shiny silver handle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base7.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base7.json new file mode 100644 index 00000000..759305ab --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base7.json @@ -0,0 +1,22 @@ +{ + "raw_description": "cup", + "seen": [ + "cylindrical cup", + "cup with handle", + "medium-sized cup", + "blue cup for drinking", + "smooth blue ceramic cup", + "abstract-patterned blue cup", + "handle-equipped drinking cup", + "ceramic cup with blue finish", + "blue cup with abstract patterns", + "light blue cup with sturdy handle", + "decorative light blue drinking cup", + "blue cylindrical cup with smooth finish" + ], + "unseen": [ + "light blue cup", + "cup with white handle", + "medium cup for liquids" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base8.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base8.json new file mode 100644 index 00000000..e2dc4830 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base8.json @@ -0,0 +1,22 @@ +{ + "raw_description": "cup", + "seen": [ + "light blue ceramic cup", + "light blue cup for liquids", + "medium blue mug with handle", + "smooth glossy light-blue cup", + "blue cup with elephant print", + "cartoon-printed blue coffee cup", + "palm-sized blue cup with handle", + "light blue cup with curved handle", + "blue drinking cup with side handle", + "cartoon-decorated blue ceramic cup", + "cylindrical cup with cartoon design", + "smooth ceramic mug with light blue color" + ], + "unseen": [ + "blue cup", + "ceramic cup with shiny finish", + "cup with cartoon elephant print" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base9.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base9.json new file mode 100644 index 00000000..60d62f16 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/021_cup/base9.json @@ -0,0 +1,22 @@ +{ + "raw_description": "cup", + "seen": [ + "white cup", + "small cup for liquids", + "cute white cup with handle", + "cup with black circular eyes", + "cup with brown curved handle", + "white cup with playful design", + "cup with smooth rounded handle", + "cup with yellow dome decoration", + "tiny cup with duck-like features", + "white ceramic cup with decorations", + "cup featuring yellow knob and black dots", + "cup with rounded edges and looped handle" + ], + "unseen": [ + "white cylinder-shaped cup", + "ceramic cup with brown handle", + "small cup with yellow decoration" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/022_cup-with-liquid/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/022_cup-with-liquid/base0.json new file mode 100644 index 00000000..79307371 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/022_cup-with-liquid/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "cup with liquid", + "seen": [ + "blue cup with yellow liquid", + "cup with smooth blue surface", + "light blue cup cylinder shape", + "medium hand-held drinking cup", + "medium round blue drinking cup", + "cup with shiny light blue body", + "yellow liquid visible in open top", + "round container for yellow liquid", + "cylindrical blue cup holds liquid", + "yellow liquid inside light blue cup", + "cup with smooth bright blue coating", + "yellow fluid sitting inside blue cup" + ], + "unseen": [ + "blue glossy finish cup", + "cup with curved light blue walls", + "open-topped cup containing yellow drink" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/023_tissue-box/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/023_tissue-box/base0.json new file mode 100644 index 00000000..51fb3698 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/023_tissue-box/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tissue box", + "seen": [ + "rectangular tissue box", + "red patterned tissue box", + "box with oval tissue slot", + "tissue box with oval slit", + "tabletop tissue holder box", + "medium box with red designs", + "medium black box with tissues", + "smooth box with tissue opening", + "black box with white tissue slot", + "black tissue box with red accents", + "rectangular black box for tissues", + "smooth black rectangular tissue box" + ], + "unseen": [ + "black tissue box", + "black box with oval opening", + "black box with white tissue dispenser" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/023_tissue-box/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/023_tissue-box/base1.json new file mode 100644 index 00000000..619e46f9 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/023_tissue-box/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tissue box", + "seen": [ + "wooden tissue box", + "medium tissue box", + "box with tissue slot", + "smooth wood tissue box", + "box for dispensing tissues", + "rectangular box for tissues", + "brown rectangular tissue box", + "dark brown wooden tissue box", + "tissue holder with oval slot", + "tissue box with white opening", + "tissue box with grain texture", + "brown box for storing tissues" + ], + "unseen": [ + "wood grain tissue box", + "medium wooden tissue holder", + "wooden box with tissue opening" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/023_tissue-box/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/023_tissue-box/base2.json new file mode 100644 index 00000000..e76e2c64 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/023_tissue-box/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tissue-box", + "seen": [ + "blue tissue-box", + "cardboard tissue-box", + "blue box with tissues", + "smooth blue tissue-box", + "tissue-box for cleaning", + "hand-sized blue tissue-box", + "medium rectangular tissue-box", + "white tissues inside blue box", + "blue tissue-box containing soft paper", + "soft tissues popping out of tissue-box", + "tissue-box made of cardboard and paper", + "rectangular tissue-box with top opening" + ], + "unseen": [ + "rectangular blue tissue-box", + "blue box with smooth texture", + "tissue-box with white tissues" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/023_tissue-box/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/023_tissue-box/base3.json new file mode 100644 index 00000000..a4ede341 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/023_tissue-box/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tissue box", + "seen": [ + "white tissue box", + "box with tissues", + "smooth tissue box", + "hand-sized tissue holder", + "white box with brown panel", + "medium-sized box for tissues", + "compact tissue box for wiping", + "rectangular hand-held tissue box", + "tissue box with soft white sheets", + "rectangular white box with tissues", + "rectangular prism tissue container", + "tissue box with brown dispensing side" + ], + "unseen": [ + "white box with visible tissues", + "box of soft white paper tissues", + "white box with pullable tissues" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/023_tissue-box/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/023_tissue-box/base4.json new file mode 100644 index 00000000..bbf75754 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/023_tissue-box/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tissue-box", + "seen": [ + "box for tissues", + "rectangular tissue-box", + "box with soft white tissues", + "smooth light gray tissue-box", + "box with white folded tissue", + "simple rectangular tissue-box", + "soft white tissues in gray box", + "gray tissue-box with top opening", + "cardboard box with tissue opening", + "medium gray box with soft tissues", + "rectangular box with tissues inside", + "white paper tissues popping out box" + ], + "unseen": [ + "gray tissue-box", + "medium tissue-box", + "light gray box holding tissues" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/023_tissue-box/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/023_tissue-box/base5.json new file mode 100644 index 00000000..2b94e81b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/023_tissue-box/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tissue box", + "seen": [ + "green box", + "tissue box", + "rounded green box with tissues", + "green box with yellow border slit", + "box with white tissues sticking out", + "plastic box with yellow tissue slot", + "smooth green box with rounded edges", + "medium-sized box for holding tissues", + "glossy tissue box with smooth finish", + "green plastic box with white tissues", + "rounded rectangular tissue box design", + "green box featuring removable tissues" + ], + "unseen": [ + "small green tissue box", + "green and yellow tissue dispenser", + "hand-sized tissue box for dispensing" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/023_tissue-box/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/023_tissue-box/base6.json new file mode 100644 index 00000000..b9863a73 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/023_tissue-box/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tissue-box", + "seen": [ + "white tissue-box", + "medium-sized tissue-box", + "smooth white tissue-box", + "tissue-box with black stripe", + "tissue-box with glossy finish", + "white slanted box for tissues", + "tissue-box with a round opening", + "white tissue-box with black accent", + "white tissue-box with circular hole", + "angled tissue-box with smooth surface", + "compact tissue-box with diagonal edges", + "lightweight tissue-box for easy carrying" + ], + "unseen": [ + "white box with tissue slot", + "slanted rectangular tissue-box", + "angled tissue-box for holding tissues" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/024_scanner/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/024_scanner/base0.json new file mode 100644 index 00000000..b64a90d9 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/024_scanner/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "scanner", + "seen": [ + "black scanner", + "scanner with curved grip", + "scanner with gray accents", + "scanner for reading barcodes", + "barcode scanner with flat top", + "black scanner with gray handle", + "smooth plastic barcode scanner", + "scanner with trigger on handle", + "black and gray portable scanner", + "scanner with flat reading surface", + "scanner with ergonomic grip design", + "lightweight handheld barcode scanner" + ], + "unseen": [ + "barcode scanner", + "handheld scanner", + "compact black barcode scanner" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/024_scanner/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/024_scanner/base1.json new file mode 100644 index 00000000..80c62639 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/024_scanner/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "scanner", + "seen": [ + "black scanner", + "handheld scanner", + "matte black scanner", + "scanner with curved handle", + "small black handheld scanner", + "scanner with pointed bottom tip", + "scanner with broad top flat area", + "barcode scanner with gray accents", + "black scanner with smooth texture", + "curved black scanner with trigger", + "scanner with gray and black design", + "black scanner with gray textured tip" + ], + "unseen": [ + "compact barcode scanner", + "scanner for barcode scanning", + "scanner with wide top section" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/024_scanner/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/024_scanner/base2.json new file mode 100644 index 00000000..0d200dbd --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/024_scanner/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "scanner", + "seen": [ + "black scanner", + "barcode scanner", + "handheld scanner", + "scanner for reading barcodes", + "scanner with smooth black body", + "scanner with blue scanning area", + "hand scanner with blue lens area", + "compact black scanner for easy grip", + "black plastic scanner with blue trim", + "L-shaped scanner for barcode reading", + "smooth black scanner with blue stripe", + "scanner with curved top and flat bottom" + ], + "unseen": [ + "small scanner fits in hand", + "black scanner with ergonomic handle", + "handheld scanner with blue activation trigger" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/024_scanner/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/024_scanner/base3.json new file mode 100644 index 00000000..b23400ab --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/024_scanner/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "scanner", + "seen": [ + "barcode scanner", + "small scanner system", + "small handheld scanner", + "compact plastic barcode scanner", + "scanner with smooth plastic body", + "barcode scanner with curved handle", + "scanner with rectangular black end", + "gray scanner with ergonomic handle", + "light gray scanner with blue button", + "gray scanner with black scanning head", + "scanner body with blue trigger button", + "scanner handle with slightly curved design" + ], + "unseen": [ + "light gray scanner", + "scanner with black tip", + "light gray scanner with smooth finish" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/024_scanner/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/024_scanner/base4.json new file mode 100644 index 00000000..878609c8 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/024_scanner/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "scanner", + "seen": [ + "barcode scanner", + "handheld scanner", + "gun-shaped scanner", + "scanner for barcodes", + "medium handheld scanner", + "scanner with scanning head", + "scanner with textured grip", + "yellow scanner with buttons", + "yellow and black code scanner", + "scanner with black rubber grip", + "barcode scanner with yellow body", + "rubber-grip yellow barcode scanner" + ], + "unseen": [ + "trigger scanner", + "yellow and black scanner", + "plastic yellow gun-shaped scanner" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/025_chips-tub/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/025_chips-tub/base0.json new file mode 100644 index 00000000..3c84aa70 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/025_chips-tub/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "chips-tub", + "seen": [ + "pink chips-tub", + "medium-sized chips-tub", + "printed pink chips-tub", + "pink tub with yellow top", + "cylindrical pink chips-tub", + "snack-sized pink chips-tub", + "chips-tub for holding snacks", + "chips-tub with smooth surface", + "chips-tub with flat gray bottom", + "rounded chips-tub with pink body", + "pink chips-tub for chips storage", + "chips-tub with colorful printed design" + ], + "unseen": [ + "yellow-lid chips-tub", + "chips-tub with sealed yellow lid", + "chips-tub with white and yellow label" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/025_chips-tub/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/025_chips-tub/base1.json new file mode 100644 index 00000000..0fd0dc05 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/025_chips-tub/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "chips tub", + "seen": [ + "blue chips tub", + "medium-sized chips tub", + "yellow lid on chips tub", + "chips tub with yellow lid", + "blue tub for storing chips", + "chip container with yellow top", + "chips tub with smooth flat bottom", + "smooth blue and yellow plastic tub", + "potato chip tub with printed design", + "yellow and blue cylindrical chips tub", + "rounded chips tub with labeled surface", + "cylindrical chips tub with bright colors" + ], + "unseen": [ + "Lay's Stax chips tub", + "snack tub with blue body", + "chips tub with white printed label" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/025_chips-tub/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/025_chips-tub/base2.json new file mode 100644 index 00000000..1bbbea04 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/025_chips-tub/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "chips tub", + "seen": [ + "orange chips tub", + "cylindrical chips tub", + "medium-sized chips tub", + "sealed orange chips tub", + "handheld orange chips tub", + "snack tub with rounded ends", + "bright orange snack container", + "orange tube for storing chips", + "chips tub with smooth texture", + "tub with printed chip graphics", + "light cardboard chips container", + "bright orange cylinder for snacks" + ], + "unseen": [ + "smooth orange tub with lid", + "chips tub with yellow label", + "chips tub with yellow and white graphics" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/025_chips-tub/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/025_chips-tub/base3.json new file mode 100644 index 00000000..1fb4dac2 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/025_chips-tub/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "chips tub", + "seen": [ + "yellow chips tub", + "hand-held chips tub", + "glossy yellow snack tub", + "cardboard tub for chips", + "chips tub with sealed top", + "yellow tub with plastic lid", + "tub for holding potato chips", + "chips tub with bright design", + "bright yellow cylindrical chips tub", + "chips tub with smooth glossy surface", + "cylinder tub with potato chips label", + "yellow tub with branding and chips picture" + ], + "unseen": [ + "smooth yellow chips tub", + "medium-sized tub for chips", + "yellow cylinder for storing chips" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/026_pet-collar/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/026_pet-collar/base0.json new file mode 100644 index 00000000..9d5e023f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/026_pet-collar/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "pet-collar", + "seen": [ + "black pet-collar", + "nylon pet-collar", + "black band collar", + "circular pet-collar", + "adjustable pet-collar", + "plastic buckle collar", + "medium nylon pet-collar", + "collar for pets with buckle", + "adjustable black strap collar", + "pet-collar with circular shape", + "medium black collar with buckle", + "black collar with rugged texture" + ], + "unseen": [ + "smooth strap pet-collar", + "gray-accented pet-collar", + "pet-collar with gray buckle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/026_pet-collar/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/026_pet-collar/base1.json new file mode 100644 index 00000000..edb5ea67 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/026_pet-collar/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "pet-collar", + "seen": [ + "smooth pet-collar", + "rounded pet-collar", + "red band pet-collar", + "pet-collar for pets", + "soft inner pet-collar", + "flexible red pet-collar", + "pet-collar made of fabric", + "red pet-collar with a clasp", + "pet-collar with black buckle", + "black latch on red pet-collar", + "medium-sized circular pet-collar", + "collar with red and black design" + ], + "unseen": [ + "red pet-collar", + "durable pet-collar", + "adjustable pet-collar" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/026_pet-collar/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/026_pet-collar/base2.json new file mode 100644 index 00000000..11a0d9fa --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/026_pet-collar/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "pet-collar", + "seen": [ + "pet-collar with silver bell", + "blue belt-shaped pet-collar", + "collar with shiny metal bell", + "blue round collar with buckle", + "smooth blue adjustable collar", + "collar with round silver bell", + "blue circular band pet-collar", + "plastic blue collar with clasp", + "blue smooth collar with buckle", + "blue pet-collar for cats or dogs", + "adjustable collar with white accents", + "blue collar with white plastic buckle" + ], + "unseen": [ + "blue pet-collar", + "small adjustable pet-collar", + "pet-collar with shiny silver bell" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/026_pet-collar/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/026_pet-collar/base3.json new file mode 100644 index 00000000..88152c18 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/026_pet-collar/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "pet-collar", + "seen": [ + "orange pet-collar", + "smooth pet-collar", + "neck strap for pets", + "adjustable pet-collar", + "plastic orange collar", + "durable orange collar", + "pet-collar for animals", + "collar with metal clasp", + "collar with adjustable fit", + "pet-collar with small holes", + "round orange collar with buckle", + "orange collar with silver buckle" + ], + "unseen": [ + "medium pet-collar with holes", + "pet-collar strap with fastener", + "bright orange circular pet-collar" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/027_table-tennis/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/027_table-tennis/base0.json new file mode 100644 index 00000000..e6d08b19 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/027_table-tennis/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "table-tennis", + "seen": [ + "dark gray ball", + "table-tennis ball", + "plastic ping pong ball", + "tiny round ping pong ball", + "round ball for table-tennis", + "small gray table-tennis ball", + "dark gray spherical game ball", + "small lightweight plastic ball", + "perfectly smooth dark gray ball", + "small smooth sphere for ping pong", + "round dark gray table-tennis ball", + "smooth lightweight table-tennis ball" + ], + "unseen": [ + "ping pong ball", + "dark gray table-tennis ball", + "perfect sphere for table-tennis games" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/027_table-tennis/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/027_table-tennis/base1.json new file mode 100644 index 00000000..4b3661fb --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/027_table-tennis/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "table-tennis", + "seen": [ + "small orange ball", + "tiny table-tennis ball", + "smooth orange sport ball", + "orange sphere for table tennis", + "bright orange table-tennis ball", + "sphere with bright orange color", + "small orange ball used for games", + "smooth and lightweight orange ball", + "bright orange lightweight sports ball", + "round lightweight ball for table tennis", + "smooth plastic orange table-tennis ball", + "perfectly round orange table-tennis ball" + ], + "unseen": [ + "small orange sphere", + "orange table-tennis ball", + "light palm-sized orange ball" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/028_roll-paper/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/028_roll-paper/base0.json new file mode 100644 index 00000000..13f9c342 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/028_roll-paper/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "roll paper", + "seen": [ + "paper roll", + "white roll paper", + "medium roll paper", + "cleaning roll paper", + "soft white roll paper", + "white roll for cleaning", + "white textured paper roll", + "soft cylindrical roll paper", + "roll paper with soft surface", + "roll paper with circular ends", + "roll paper with smooth texture", + "smooth white cylindrical roll paper" + ], + "unseen": [ + "white paper cylinder", + "cylinder-shaped paper roll", + "medium cylindrical white paper roll" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/028_roll-paper/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/028_roll-paper/base1.json new file mode 100644 index 00000000..e78e032f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/028_roll-paper/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "roll-paper", + "seen": [ + "soft roll-paper", + "roll-paper for wiping", + "cylindrical roll-paper", + "hollow center roll-paper", + "blue and white roll-paper", + "roll-paper with soft texture", + "roll-paper with hollow center", + "paper roll wrapped in plastic", + "roll-paper with smooth wrapper", + "compact blue and white roll-paper", + "white paper roll with blue wrapper", + "cylindrical roll-paper with smooth finish" + ], + "unseen": [ + "small roll-paper", + "hand-sized roll-paper", + "roll-paper wrapped in blue plastic" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/028_roll-paper/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/028_roll-paper/base2.json new file mode 100644 index 00000000..2664140b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/028_roll-paper/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "roll paper", + "seen": [ + "white roll paper", + "soft white paper roll", + "thin white roll paper", + "smooth textured roll paper", + "roll paper with thin sheets", + "white roll paper for wiping", + "roll paper with soft material", + "roll paper with smooth finish", + "white paper roll for bathrooms", + "roll paper for cleaning surfaces", + "medium-sized paper roll cylinder", + "cylindrical roll paper with sheets" + ], + "unseen": [ + "toilet paper roll", + "roll paper with perforated edges", + "cylindrical roll with hollow middle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/028_roll-paper/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/028_roll-paper/base3.json new file mode 100644 index 00000000..a68e4392 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/028_roll-paper/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "roll-paper", + "seen": [ + "yellow roll-paper", + "hand-sized roll-paper", + "cylindrical roll-paper", + "medium-sized roll-paper", + "smooth yellow roll-paper", + "paper roll for daily use", + "brown and yellow roll-paper", + "roll-paper for cleaning purposes", + "roll-paper cylinder with open core", + "yellow roll-paper with brown stripe", + "roll-paper with visible paper layers", + "yellow roll-paper with labeled surface" + ], + "unseen": [ + "roll-paper with hollow center", + "roll-paper made of thin paper", + "yellow roll-paper with brown base" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/029_olive-oil/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/029_olive-oil/base0.json new file mode 100644 index 00000000..de354dac --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/029_olive-oil/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "olive oil", + "seen": [ + "yellow bottle", + "olive oil bottle", + "bottle holding olive oil", + "yellow bottle for olive oil", + "yellow plastic oil container", + "plastic bottle with green top", + "yellow bottle with black text", + "green capped yellow oil bottle", + "smooth yellow cylindrical bottle", + "rounded yellow bottle with label", + "yellow bottle with green screw cap", + "medium yellow bottle with green lid" + ], + "unseen": [ + "olive oil inside yellow bottle", + "green lid on yellow oil bottle", + "medium size yellow plastic bottle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/029_olive-oil/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/029_olive-oil/base1.json new file mode 100644 index 00000000..5300e833 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/029_olive-oil/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "olive-oil", + "seen": [ + "green olive-oil bottle", + "olive-oil bottle easy to grip", + "dark green olive-oil container", + "olive-oil bottle with green cap", + "glass olive-oil bottle green cap", + "olive-oil bottle dark green label", + "bottle for olive-oil green design", + "glass bottle olive-oil golden text", + "rounded rectangular olive-oil bottle", + "olive-oil bottle smooth glossy surface", + "dark green rectangular olive-oil bottle", + "rectangular olive-oil bottle with rounded edges" + ], + "unseen": [ + "smooth olive-oil glass bottle", + "medium-sized bottle for olive-oil", + "olive-oil bottle with sleek finish" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/029_olive-oil/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/029_olive-oil/base2.json new file mode 100644 index 00000000..27aa3312 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/029_olive-oil/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "olive oil", + "seen": [ + "olive oil bottle", + "green olive oil bottle", + "glass bottle for olive oil", + "medium glass olive oil bottle", + "green bottle for storing olive oil", + "olive oil bottle with rounded edges", + "olive oil bottle with glossy texture", + "medium container with dark green glass", + "dark green rectangular olive oil bottle", + "rounded rectangular olive oil container", + "olive oil bottle with rectangular shape", + "dark green liquid container for olive oil" + ], + "unseen": [ + "green bottle with black cap", + "smooth dark green olive oil container", + "dark green olive oil bottle with label" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/029_olive-oil/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/029_olive-oil/base3.json new file mode 100644 index 00000000..78b77950 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/029_olive-oil/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "olive oil", + "seen": [ + "olive oil bottle", + "dark green bottle", + "glass bottle with black cap", + "dark green rectangular bottle", + "glass bottle storing olive oil", + "dark green glossy glass bottle", + "medium bottle with yellow label", + "yellow-labeled olive oil holder", + "smooth bottle with tapering neck", + "medium-sized olive oil container", + "rectangular dark green olive oil bottle", + "bottle with extra virgin olive oil label" + ], + "unseen": [ + "dark green olive oil bottle", + "glass container for olive oil", + "rectangular bottle with rounded neck" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/029_olive-oil/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/029_olive-oil/base4.json new file mode 100644 index 00000000..df8208ef --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/029_olive-oil/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "olive oil", + "seen": [ + "glossy olive oil bottle", + "bottle containing olive oil", + "hand-sized olive oil bottle", + "olive oil bottle with black cap", + "yellowish-green olive oil bottle", + "medium-sized olive oil container", + "cylindrical bottle with olive oil", + "yellow-green bottle for olive oil", + "olive oil bottle with tapered neck", + "olive oil bottle with shiny finish", + "olive oil bottle with smooth surface", + "medium-sized bottle holding olive oil" + ], + "unseen": [ + "smooth olive oil bottle", + "olive oil in plastic bottle", + "black-capped olive oil bottle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/030_drill/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/030_drill/base0.json new file mode 100644 index 00000000..0a21c0bf --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/030_drill/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "drill", + "seen": [ + "cordless drill", + "battery-operated drill", + "drill with black handle", + "drill with silver chuck", + "yellow tool for drilling", + "drill with gray metallic tip", + "yellow drill with chunky body", + "yellow drill shaped like an L", + "electric drill with smooth grip", + "compact drill with sturdy handle", + "medium-sized yellow and black drill", + "yellow plastic drill with steel tip" + ], + "unseen": [ + "yellow drill", + "handheld electric drill", + "yellow drill with black grip" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/030_drill/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/030_drill/base1.json new file mode 100644 index 00000000..695f6dc0 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/030_drill/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "drill", + "seen": [ + "gray drill", + "handheld drill", + "red button drill", + "black and red drill", + "drill with gray body", + "compact electric drill", + "corded gun-shaped drill", + "drill with battery at base", + "gray drill with black handle", + "smooth drill with ridged grip", + "drill with cylindrical black tip", + "portable drill with ergonomic grip" + ], + "unseen": [ + "plastic and metal drill", + "drill with black battery", + "gray drill with red accents" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/030_drill/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/030_drill/base2.json new file mode 100644 index 00000000..c4541b06 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/030_drill/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "drill", + "seen": [ + "blue drill", + "cordless drill", + "drill with textured grip", + "drill with orange accents", + "medium drill for home use", + "drill with cylindrical front", + "drill with spinning metal tip", + "blue plastic drill with metal tip", + "electric drill with smooth finish", + "portable drill with orange button", + "power drill with rectangular battery", + "black and blue drill with white detail" + ], + "unseen": [ + "black and blue drill", + "handheld drill with battery pack", + "medium-sized drill with sturdy grip" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/030_drill/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/030_drill/base3.json new file mode 100644 index 00000000..8182ca54 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/030_drill/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "drill", + "seen": [ + "drill machine", + "blue handheld drill", + "drill for screws and holes", + "blue drill with battery pack", + "medium drill with pistol grip", + "compact drill with sturdy handle", + "blue drill with black drill head", + "drill with visible trigger in red", + "electric drill with smooth housing", + "drill built from plastic and metal", + "blue drill with black front section", + "blue medium-sized drill with red accents" + ], + "unseen": [ + "blue drill", + "drill with black and red areas", + "handheld drill with textured grip" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/030_drill/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/030_drill/base4.json new file mode 100644 index 00000000..d4d15b47 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/030_drill/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "drill", + "seen": [ + "yellow drill", + "gun-shaped yellow drill", + "drill with beige spiral bit", + "yellow drill with red trigger", + "handheld drill with spiral bit", + "plastic and metal yellow drill", + "handheld tool for drilling holes", + "yellow body drill with gray parts", + "medium-sized yellow and purple drill", + "yellow drill with curved purple grip", + "electric drill with smooth plastic body", + "yellow drill with thumb-sized red trigger" + ], + "unseen": [ + "spiral beige drill bit on tool", + "drill with textured purple handle", + "yellow drill with flat purple base" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/030_drill/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/030_drill/base5.json new file mode 100644 index 00000000..049a2ce8 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/030_drill/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "drill", + "seen": [ + "teal drill", + "cordless teal drill", + "compact handheld drill", + "drill with black accents", + "drill with cylindrical tip", + "drill with front bit holder", + "drill tool for making holes", + "teal drill with black handle", + "T-shaped black and teal drill", + "small power drill with trigger", + "compact rubber grip teal drill", + "teal power drill with black parts" + ], + "unseen": [ + "rubber grip drill", + "lightweight hand drill", + "electric drill with smooth body" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/030_drill/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/030_drill/base6.json new file mode 100644 index 00000000..8707d331 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/030_drill/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "drill", + "seen": [ + "black drill", + "handheld black drill", + "cordless black drill", + "drill for making holes", + "lightweight black drill", + "plastic drill with metal tip", + "drill with ridged grip handle", + "small black drill with L-shape", + "black drill with rotating head", + "black drill with smooth finish", + "black drill with cylindrical top", + "L-shaped drill with battery base" + ], + "unseen": [ + "compact drill", + "drill with light blue trigger", + "smooth black drill with blue accents" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/031_jam-jar/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/031_jam-jar/base0.json new file mode 100644 index 00000000..48e24b45 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/031_jam-jar/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "jam jar", + "seen": [ + "jam jar", + "red jam jar", + "red lid jam jar", + "smooth glass jar", + "clear jar for jams", + "glass jar with red lid", + "jar with shiny red lid", + "round jar for jam storage", + "red-topped glass container", + "jam jar with vibrant red cap", + "glass container for preserves", + "glass bottle with red screw cap" + ], + "unseen": [ + "medium jar with round lid", + "medium cylindrical jam jar", + "clear jar with smooth edges" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/031_jam-jar/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/031_jam-jar/base1.json new file mode 100644 index 00000000..6c55d58a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/031_jam-jar/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "jam jar", + "seen": [ + "red jam jar", + "red jar for jam", + "cylindrical jam jar", + "yellow-topped jam jar", + "jam jar made of glass", + "jam jar with yellow lid", + "small jar for storing jam", + "small jar with tapered body", + "red jam jar with metallic top", + "compact jar of sweet preserves", + "glass jam jar with screw-on lid", + "smooth glass jar with red contents" + ], + "unseen": [ + "palm-sized jam jar", + "yellow-lidded jar holding jam", + "glass jar filled with red jam" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/031_jam-jar/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/031_jam-jar/base2.json new file mode 100644 index 00000000..ee118d44 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/031_jam-jar/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "jam jar", + "seen": [ + "red jar", + "jam jar", + "medium glass jar", + "small cylindrical jar", + "glass jar holding jam", + "red jar with white cap", + "jam jar for storing food", + "clear jar with red contents", + "red jar with white screw lid", + "red jam jar with strawberries", + "round jar with strawberry design", + "cylindrical jar with smooth surface" + ], + "unseen": [ + "hand-sized glass jar", + "red jar with fruit label", + "glass jar with white lid" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/031_jam-jar/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/031_jam-jar/base3.json new file mode 100644 index 00000000..a3d2842b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/031_jam-jar/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "jam jar", + "seen": [ + "red jam jar", + "white-labeled jam jar", + "jam jar with textured lid", + "jar with black plastic lid", + "small jam jar fits in hand", + "compact jar for jam storage", + "dark red rectangular jam jar", + "dark red jar with white wrap", + "red jam jar with rounded corners", + "smooth glass jar with red content", + "red rectangular jar with black lid", + "glass jar with visible red contents" + ], + "unseen": [ + "small glass jar for jam", + "jam jar with white label", + "jam container with glossy finish" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/031_jam-jar/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/031_jam-jar/base4.json new file mode 100644 index 00000000..5ccaaac4 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/031_jam-jar/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "jam jar", + "seen": [ + "red jam jar", + "deep red jam jar", + "jar with fruit label", + "small clear glass jar", + "red and white jam jar", + "cylindrical jar for jam", + "jam jar with striped lid", + "jam jar with plastic lid", + "glass jar of strawberry jam", + "jam jar with white lettering", + "smooth jar with red contents", + "striped red and white top jar" + ], + "unseen": [ + "red jar with white label", + "glass jar with red contents", + "medium-sized jar of preserves" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/032_screwdriver/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/032_screwdriver/base0.json new file mode 100644 index 00000000..6daec610 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/032_screwdriver/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "screwdriver", + "seen": [ + "manual screwdriver", + "black and yellow tool", + "slim shaft screwdriver", + "rubber handle screwdriver", + "yellow and black screwdriver", + "rubber and metal screwdriver", + "yellow rubber grip screwdriver", + "long screwdriver with black handle", + "small black and yellow screwdriver", + "screwdriver with pointed steel end", + "screwdriver with textured black grip", + "compact screwdriver with sturdy handle" + ], + "unseen": [ + "handheld screwdriver", + "steel shaft screwdriver", + "steel screwdriver with pointed tip" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/033_fork/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/033_fork/base0.json new file mode 100644 index 00000000..e2e740b0 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/033_fork/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "fork", + "seen": [ + "table fork", + "silver fork", + "shiny metal fork", + "prong-ended fork", + "medium-sized fork", + "long-handled fork", + "silvery food fork", + "smooth silver fork", + "silver dining fork", + "simple silver fork", + "fork with long handle", + "fork with four prongs" + ], + "unseen": [ + "metal fork", + "metal utensil fork", + "fork with flat prongs" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/034_knife/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/034_knife/base0.json new file mode 100644 index 00000000..5a7ba0a1 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/034_knife/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "knife", + "seen": [ + "metal knife", + "silver knife", + "medium cutting knife", + "knife with wooden handle", + "sharp stainless steel knife", + "knife blade and brown handle", + "kitchen knife with sharp edge", + "long blade knife with wood grip", + "silver knife with smooth finish", + "knife designed for kitchen work", + "medium-sized silver kitchen knife", + "silver cutting knife with curved blade" + ], + "unseen": [ + "brown-handled knife", + "knife for slicing and cutting", + "knife with textured wooden grip" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/035_apple/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/035_apple/base0.json new file mode 100644 index 00000000..357fa250 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/035_apple/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "apple", + "seen": [ + "ripe red apple", + "shiny red apple", + "smooth bright red apple", + "small round edible apple", + "red fruit called an apple", + "palm-sized shiny red apple", + "fresh apple with shiny skin", + "red apple with subtle dimples", + "small fruit shaped like apple", + "organic apple with smooth skin", + "apple with slightly uneven shape", + "rounded apple with glossy surface" + ], + "unseen": [ + "red apple", + "shiny red rounded apple", + "bright red apple with smooth texture" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/035_apple/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/035_apple/base1.json new file mode 100644 index 00000000..607d849a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/035_apple/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "apple", + "seen": [ + "red apple", + "dark red apple", + "round dark red apple", + "red apple with tiny stem", + "hand-sized dark red apple", + "apple with smooth waxy skin", + "apple with dents and red skin", + "apple with smooth dark red skin", + "rounded fruit with deep red color", + "dark red apple perfect for holding", + "medium-sized apple with shiny texture", + "red apple with slightly uneven surface" + ], + "unseen": [ + "small shiny red apple", + "dark red fruit with rounded shape", + "red apple with slightly bumpy bottom" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/036_cabinet/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/036_cabinet/base0.json new file mode 100644 index 00000000..dc2a0f06 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/036_cabinet/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "cabinet", + "seen": [ + "wooden cabinet", + "storage cabinet", + "light wood cabinet", + "rectangular cabinet", + "medium-sized cabinet", + "gray and wood cabinet", + "smooth-surface cabinet", + "light wood top cabinet", + "cabinet with gray sides", + "cabinet with angled sides", + "rectangular slanted cabinet", + "cabinet with metal reinforcements" + ], + "unseen": [ + "slanted wooden cabinet", + "cabinet with smooth finish", + "cabinet with light wood panels" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/037_box/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/037_box/base0.json new file mode 100644 index 00000000..f83468ee --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/037_box/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "box", + "seen": [ + "brown box", + "rectangular box", + "cardboard-like box", + "smooth rectangular box", + "box with smooth textures", + "brown box with gray panels", + "medium box with brown edges", + "medium-sized box for storage", + "box with brown top and bottom", + "box with brown corners and edges", + "gray and brown rectangular prism box", + "rectangular box with dark and light areas" + ], + "unseen": [ + "rectangular storage box", + "box with dark gray sides", + "medium box, brown-gray rectangular shape" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/038_milk-box/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/038_milk-box/base0.json new file mode 100644 index 00000000..95176662 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/038_milk-box/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "milk box", + "seen": [ + "white milk box", + "carton milk box", + "blue lid milk box", + "hand-sized milk box", + "blue-striped milk box", + "milk box with pouring spout", + "milk box with smooth surface", + "milk box with branding and text", + "milk box with slanted top design", + "white milk box with glossy finish", + "milk box with laminated paper surface", + "milk box with nutritional info printed" + ], + "unseen": [ + "blue and white milk box", + "milk box with blue letters", + "rectangular milk box with slanted top" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/038_milk-box/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/038_milk-box/base1.json new file mode 100644 index 00000000..f02b8291 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/038_milk-box/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "milk-box", + "seen": [ + "blue milk-box", + "small milk-box", + "carton milk-box", + "angled top milk-box", + "rectangular milk-box", + "blue carton milk-box", + "milk-box with white top", + "white and blue milk-box", + "milk-box with blue sides", + "milk-box with sharp edges", + "milk-box with angled spout", + "milk-box for holding liquids" + ], + "unseen": [ + "glossy blue milk-box", + "milk-box with smooth surface", + "milk-box with printed design" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/038_milk-box/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/038_milk-box/base2.json new file mode 100644 index 00000000..9211484e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/038_milk-box/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "milk box", + "seen": [ + "milk box", + "blue green carton", + "milk box holds liquid", + "carton for milk storage", + "milk carton colorful design", + "compact rectangular milk box", + "colorful milk box rectangular", + "hand-sized blue green milk box", + "rectangular smooth milk carton", + "milk box rectangular prism shape", + "blue green white smooth container", + "smooth milk box with glossy sections" + ], + "unseen": [ + "rectangular milk container", + "milk box with glossy finish", + "carton with blue green white colors" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/038_milk-box/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/038_milk-box/base3.json new file mode 100644 index 00000000..2d87d1e9 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/038_milk-box/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "milk box", + "seen": [ + "white milk box", + "medium milk box", + "printed milk box", + "handheld milk box", + "cardboard milk box", + "rectangular milk box", + "smooth white milk box", + "red and white milk box", + "milk box with red print", + "milk box with sealed top", + "milk box with label and spout", + "milk box with folded spout top" + ], + "unseen": [ + "milk box with folded design", + "white milk box with red accents", + "milk box used for liquid storage" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base0.json new file mode 100644 index 00000000..65294ccb --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "mug", + "seen": [ + "black mug", + "black ceramic mug", + "ceramic mug smooth outside", + "medium black mug with handle", + "mug with curved black handle", + "palm-sized mug glossy black body", + "medium mug smooth rounded handle", + "cylindrical mug with black finish", + "black mug glossy ceramic material", + "drinking mug shiny black exterior", + "black mug with rounded sturdy handle", + "glossy black mug medium-sized for drinking" + ], + "unseen": [ + "rounded mug with glossy shine", + "mug for liquids black ceramic body", + "cylindrical drinking mug black color" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base1.json new file mode 100644 index 00000000..fd192738 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "mug", + "seen": [ + "black mug", + "drinking mug", + "black coffee mug", + "hand-sized black mug", + "black mug with handle", + "cylindrical black mug", + "black mug for liquids", + "mug with rounded handle", + "solid black ceramic mug", + "smooth black ceramic mug", + "mug with thick sturdy handle", + "black mug with smooth surface" + ], + "unseen": [ + "ceramic mug", + "matte black mug", + "medium black mug" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base10.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base10.json new file mode 100644 index 00000000..3d6dddfd --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base10.json @@ -0,0 +1,22 @@ +{ + "raw_description": "mug", + "seen": [ + "brown mug", + "ceramic mug", + "cylindrical mug", + "brown drink mug", + "smooth brown mug", + "brown cup-shaped mug", + "brown mug with handle", + "solid ceramic brown mug", + "medium-sized ceramic mug", + "brown ceramic mug for hot drinks", + "medium mug for holding beverages", + "smooth cylindrical mug with handle" + ], + "unseen": [ + "brown glossy mug", + "mug with curved handle", + "smooth brown ceramic drink mug" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base11.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base11.json new file mode 100644 index 00000000..8b6d20cb --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base11.json @@ -0,0 +1,22 @@ +{ + "raw_description": "mug", + "seen": [ + "blue mug", + "ceramic mug", + "gradient ceramic mug", + "blue mug fading to tan", + "mug with gradient color", + "mug with smooth surface", + "blue and tan colored mug", + "smooth ceramic coffee mug", + "blue mug for holding drinks", + "medium blue mug with handle", + "rounded mug with smooth finish", + "cylindrical mug with curved handle" + ], + "unseen": [ + "medium-sized blue mug", + "mug with attached handle", + "light tan inside blue mug" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base12.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base12.json new file mode 100644 index 00000000..6cc8b18a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base12.json @@ -0,0 +1,22 @@ +{ + "raw_description": "mug", + "seen": [ + "white mug", + "ceramic mug", + "white mug with handle", + "smooth glossy ceramic mug", + "simple white palm-sized mug", + "mug with clean glossy surface", + "white ceramic mug for beverages", + "drinking mug with sturdy handle", + "cylindrical mug with curved handle", + "rounded mug in smooth white finish", + "curved handle on white ceramic mug", + "white drinking mug with smooth texture" + ], + "unseen": [ + "white cup-shaped mug", + "medium mug for holding drinks", + "plain white ceramic mug for liquids" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base2.json new file mode 100644 index 00000000..776c2a82 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "mug", + "seen": [ + "brown mug", + "ceramic mug", + "light brown mug", + "medium-sized mug", + "hand-held ceramic mug", + "mug with curved handle", + "brown mug for drinking", + "mug for holding liquid", + "light brown cylindrical mug", + "light brown mug with handle", + "drinking mug with smooth body", + "brown medium-sized ceramic mug" + ], + "unseen": [ + "cylindrical mug", + "smooth ceramic mug", + "mug with smooth texture" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base3.json new file mode 100644 index 00000000..394a756e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "mug", + "seen": [ + "beige mug", + "mug with prints", + "medium light mug", + "curved handle mug", + "smooth ceramic mug", + "mug for hot drinks", + "colorful cartoon mug", + "mug with cartoon design", + "mug with glossy texture", + "medium mug fits in hand", + "ceramic mug with cartoons", + "mug with yellow chicken image" + ], + "unseen": [ + "beige coffee mug", + "printed beige mug", + "cylindrical mug with handle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base4.json new file mode 100644 index 00000000..907f2700 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "mug", + "seen": [ + "white mug", + "ceramic mug", + "smooth white mug", + "mug with blue print", + "mug for drinking liquids", + "printed white ceramic mug", + "medium mug with blue design", + "white mug with glossy finish", + "white ceramic mug with handle", + "hand-sized mug with blue artwork", + "cylindrical mug with curved handle", + "white drinking mug with smooth surface" + ], + "unseen": [ + "medium-sized white mug", + "mug shaped like a cylinder", + "mug with rounded bottom and handle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base5.json new file mode 100644 index 00000000..82eb0659 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "mug", + "seen": [ + "blue mug", + "mug with handle", + "cylindrical blue mug", + "mug with loop handle", + "standard blue tea mug", + "bright blue coffee mug", + "mug with glossy surface", + "blue mug for hot drinks", + "medium-sized drinking mug", + "blue mug with solid texture", + "round mug with curved handle", + "handle-mounted blue drinking mug" + ], + "unseen": [ + "smooth shiny blue mug", + "drinking mug made of ceramic", + "lightweight mug for beverages" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base6.json new file mode 100644 index 00000000..8cfbdd3a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "mug", + "seen": [ + "ceramic mug", + "medium beige mug", + "beige ceramic mug", + "mug with rounded handle", + "smooth ceramic beige mug", + "light beige drinking mug", + "beige mug handle on side", + "medium mug with smooth texture", + "ceramic mug smooth beige surface", + "beige mug polished ceramic build", + "light beige ceramic drinking mug", + "hand-sized ceramic cylindrical mug" + ], + "unseen": [ + "beige mug", + "cylindrical mug with handle", + "mug beige smooth cylinder shape" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base7.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base7.json new file mode 100644 index 00000000..c2bfa6e6 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base7.json @@ -0,0 +1,22 @@ +{ + "raw_description": "mug", + "seen": [ + "medium beige mug", + "smooth ceramic mug", + "mug with side handle", + "mug with round handle", + "mug with rounded base", + "beige mug used for drinks", + "mug with glossy beige finish", + "mug with logo design on side", + "cylindrical beige ceramic mug", + "ceramic mug with printed logo", + "glossy beige ceramic drinking mug", + "cylindrical coffee mug with handle" + ], + "unseen": [ + "beige mug", + "ceramic mug", + "beige coffee mug" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base8.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base8.json new file mode 100644 index 00000000..fb0753c5 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base8.json @@ -0,0 +1,22 @@ +{ + "raw_description": "mug", + "seen": [ + "logo mug", + "green mug", + "ceramic mug", + "green drinking mug", + "starbucks green mug", + "smooth dark green mug", + "medium mug with handle", + "dark green ceramic cup", + "round mug with one handle", + "ceramic mug for hot drinks", + "Starbucks branded coffee mug", + "medium-sized smooth green Starbucks mug" + ], + "unseen": [ + "mug with white logo print", + "dark mug with a curved handle", + "cylindrical mug with Starbucks logo" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base9.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base9.json new file mode 100644 index 00000000..333e56b4 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/039_mug/base9.json @@ -0,0 +1,22 @@ +{ + "raw_description": "mug", + "seen": [ + "white mug", + "ceramic mug", + "plain glossy mug", + "hand-sized ceramic mug", + "medium-sized white mug", + "glossy mug for beverages", + "basic mug in solid white", + "medium white mug with handle", + "white mug with rounded handle", + "smooth and plain drinking mug", + "cylindrical mug made of ceramic", + "smooth white mug with curved handle" + ], + "unseen": [ + "white coffee mug", + "medium mug for hot drinks", + "simple white cylindrical mug" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/040_rack/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/040_rack/base0.json new file mode 100644 index 00000000..4aa143b2 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/040_rack/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "rack", + "seen": [ + "metal rack", + "smooth rack", + "dark gray rack", + "dark gray angled rack", + "smooth dark gray rack", + "rack with angular body", + "rack with two rods sticking out", + "metallic rack with smooth finish", + "medium rack with protruding rods", + "metallic angular rack with rod pair", + "angled rack with long and short rods", + "angular rack with slanted support rod" + ], + "unseen": [ + "medium-sized rack", + "dark gray rack with flat back", + "rack with straight and slanted rods" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base0.json new file mode 100644 index 00000000..dede26a7 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "shoe", + "seen": [ + "black and white shoe", + "shoe with curved sole", + "synthetic leather shoe", + "average adult-sized shoe", + "wearable foot protector shoe", + "comfortable shoe for walking", + "black shoe with white accents", + "shoe made of rubber and leather", + "dual-tone shoe with sleek design", + "medium-sized black and white shoe", + "rubber-soled shoe with lace-up top", + "shoe with smooth top and grippy sole" + ], + "unseen": [ + "black shoe with white midsole", + "black shoe with curved white sole", + "shoe with padded collar and tongue" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base1.json new file mode 100644 index 00000000..16169811 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "shoe", + "seen": [ + "rubber sole shoe", + "shoe for walking", + "medium white shoe", + "smooth white shoe", + "white sneaker shoe", + "shoe with lace holes", + "shoe with curved sole", + "synthetic leather shoe", + "shoe with textured sole", + "shoe with patterned bottom", + "white shoe with rubber sole", + "medium-sized rounded white shoe" + ], + "unseen": [ + "white shoe", + "medium adult shoe", + "rounded white shoe" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base2.json new file mode 100644 index 00000000..df5fbcd7 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "shoe", + "seen": [ + "white shoe", + "sports shoe", + "lace-up shoe", + "brown sole shoe", + "rounded-toe gym shoe", + "footwear with white top", + "black-striped athletic shoe", + "lace-up black and white shoe", + "white shoe with rubber bottom", + "shoe with textured brown sole", + "white shoe with black accents", + "shoe with smooth leather upper" + ], + "unseen": [ + "shoe with black stripes", + "shoe with striped design", + "white leather shoe with gum sole" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base3.json new file mode 100644 index 00000000..34f2108c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "shoe", + "seen": [ + "white shoe", + "medium-sized shoe", + "rounded-top white shoe", + "shoe with black accents", + "smooth surface white shoe", + "white shoe with curved top", + "white shoe with black lines", + "sports shoe white and black", + "synthetic leather white shoe", + "white shoe with three stripes", + "rubber-soled black-striped shoe", + "medium shoe with synthetic material" + ], + "unseen": [ + "shoe with ribbed sole", + "black-striped white shoe", + "shoe with black stripe design" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base4.json new file mode 100644 index 00000000..8428341d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "shoe", + "seen": [ + "white and green shoe", + "shoe with beige sole", + "green lace-up sneaker", + "footwear with padded heel", + "rubber-soled green sneaker", + "shoe with thick ribbed sole", + "green upper white logo shoe", + "smooth green shoe with laces", + "white-laced green running shoe", + "medium-sized green walking shoe", + "shoe with synthetic fabric body", + "sports shoe green with beige sole" + ], + "unseen": [ + "green shoe", + "running shoe with laces", + "beige-soled walking sneaker" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base5.json new file mode 100644 index 00000000..3ab10a5e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "shoe", + "seen": [ + "brown shoe", + "rounded front shoe", + "smooth leather shoe", + "shoe with lace holes", + "shoe for everyday wear", + "dark brown leather shoe", + "shoe with grooved bottom", + "rounded tip leather shoe", + "medium-sized walking shoe", + "shoe with thick rubber sole", + "dark shoe with grooves on sole", + "shoe for walking with rubber grip" + ], + "unseen": [ + "casual brown shoe", + "brown shoe with sturdy sole", + "brown footwear with smooth texture" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base6.json new file mode 100644 index 00000000..6a1fc2df --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "shoe", + "seen": [ + "lace-up sneaker", + "rubber sole shoe", + "rounded front shoe", + "lace-up canvas shoe", + "white shoe with red tag", + "medium-sized adult shoe", + "white shoe with flat sole", + "shoe with flat rubber sole", + "classic sneaker design shoe", + "canvas shoe with grooved sole", + "white sneaker with red accents", + "comfortable white and red shoe" + ], + "unseen": [ + "shoe with adjustable laces", + "white shoe with tongue logo", + "walking shoe with grip sole" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base7.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base7.json new file mode 100644 index 00000000..cf7d3371 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base7.json @@ -0,0 +1,22 @@ +{ + "raw_description": "shoe", + "seen": [ + "light brown shoe", + "plastic brown shoe", + "shoe with open straps", + "medium-sized brown shoe", + "shoe with textured bottom", + "shoe with colorful stripes", + "brown shoe with strap loops", + "small brown footwear replica", + "shoe with rounded and open form", + "rounded shoe with visible straps", + "light brown shoe with white sole", + "shoe with red green white pattern" + ], + "unseen": [ + "smooth plastic shoe", + "shoe model with ribbed sole", + "brown shoe with green red stripes" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base8.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base8.json new file mode 100644 index 00000000..9bb533f9 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base8.json @@ -0,0 +1,22 @@ +{ + "raw_description": "shoe", + "seen": [ + "sports shoe", + "red sneaker", + "medium red shoe", + "single red shoe", + "grooved sole red shoe", + "lace-up red sports shoe", + "red shoe with white sole", + "smooth-textured red shoe", + "white bottom sports shoe", + "red shoe rounded at front", + "shoe with flat rubber sole", + "footwear with red top white sole" + ], + "unseen": [ + "red shoe", + "synthetic shoe upper", + "red sneaker medium sized" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base9.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base9.json new file mode 100644 index 00000000..da1d412a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/041_shoe/base9.json @@ -0,0 +1,22 @@ +{ + "raw_description": "shoe", + "seen": [ + "white shoe", + "shoe with lace-up front", + "sleek white walking shoe", + "medium-sized casual shoe", + "shoe for everyday walking", + "shoe with black heel patch", + "white shoe with rounded toe", + "white shoe with curved sole", + "shoe with black back detail", + "synthetic leather white shoe", + "soft rubber-soled white shoe", + "white and black combination shoe" + ], + "unseen": [ + "shoe with rubber bottom", + "black-accented white shoe", + "smooth white shoe with laces" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/043_book/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/043_book/base0.json new file mode 100644 index 00000000..156a36ed --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/043_book/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "book", + "seen": [ + "closed blue book", + "hardcover blue book", + "book with blue cover", + "white-edged blue book", + "blue book for reading", + "medium rectangular book", + "book with red-lined pages", + "blue book with white spine", + "flat rectangular book cover", + "smooth blue rectangular book", + "book with visible paper edges", + "closed book with visible pages" + ], + "unseen": [ + "blue book", + "blue book with embossed details", + "medium-sized book with smooth cover" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/043_book/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/043_book/base1.json new file mode 100644 index 00000000..b712b780 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/043_book/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "book", + "seen": [ + "black book", + "medium-sized book", + "black hardcover book", + "hardcover black book", + "book with orange text", + "rectangular black book", + "book with smooth cover", + "book with visible spine", + "flat rectangular black book", + "rectangular book with sharp edges", + "black book with rough inner pages", + "book with sturdy rectangular shape" + ], + "unseen": [ + "orange text book", + "spine with orange letters", + "black book with white logo" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/044_microwave/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/044_microwave/base0.json new file mode 100644 index 00000000..f19ea754 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/044_microwave/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "microwave", + "seen": [ + "microwave", + "gray microwave", + "medium microwave", + "microwave with knobs", + "countertop microwave", + "rectangular microwave", + "yellow-sided microwave", + "microwave with textured top panel", + "gray microwave with two control knobs", + "microwave with patterned top and knobs", + "yellow-panel microwave with smooth sides", + "gray and yellow microwave with rounded edges" + ], + "unseen": [ + "medium gray microwave with top vents", + "gray and yellow rectangular microwave", + "medium-sized rectangular microwave with textured top" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/044_microwave/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/044_microwave/base1.json new file mode 100644 index 00000000..7d6d3086 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/044_microwave/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "microwave", + "seen": [ + "microwave", + "microwave with glass top", + "rectangular gray microwave", + "microwave for heating food", + "dark gray kitchen microwave", + "microwave with smooth surface", + "dark gray slanted microwave box", + "medium microwave with slanted sides", + "gray and glass microwave for cooking", + "microwave with glossy dark gray finish", + "gray appliance with transparent glass top", + "rectangular microwave with clear glass panel" + ], + "unseen": [ + "dark gray microwave", + "kitchen microwave with slanted shape", + "medium-sized microwave with angled walls" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/045_sand-clock/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/045_sand-clock/base0.json new file mode 100644 index 00000000..fe820f5a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/045_sand-clock/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "sand-clock", + "seen": [ + "sand-clock with white sand", + "sand-clock with smooth glass", + "sand-clock with polished finish", + "hourglass made of glass and wood", + "golden hourglass with circular ends", + "small round-based golden sand-clock", + "golden sand-clock with white center", + "golden sand-clock with flowing sand", + "compact sand-clock with sturdy design", + "hand-sized hourglass for time measurement", + "hourglass with golden supports and white sand", + "decorative sand-clock with bright yellow frame" + ], + "unseen": [ + "small hourglass", + "golden sand-clock", + "sand-clock with curved frame" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/045_sand-clock/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/045_sand-clock/base1.json new file mode 100644 index 00000000..284f6bc2 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/045_sand-clock/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "sand clock", + "seen": [ + "wooden sand clock", + "small hourglass timer", + "rounded brown sand clock", + "two-sided brown sand clock", + "glass hourglass with wooden ends", + "handheld sand clock with white sand", + "hourglass encased in wooden cylinder", + "light brown sand clock with white sand", + "compact sand timer with wooden supports", + "glass hourglass inside brown wooden frame", + "delicate sand clock made of wood and glass", + "cylindrical sand clock with glass hourglass" + ], + "unseen": [ + "brown sand clock", + "smooth sand clock with glass center", + "wood frame with white sand hourglass" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/045_sand-clock/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/045_sand-clock/base3.json new file mode 100644 index 00000000..05df5eff --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/045_sand-clock/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "sand clock", + "seen": [ + "blue sand clock", + "hourglass timer", + "hand-sized sand clock", + "small blue and white sand clock", + "timer with blue and white bulbs", + "sand clock with flowing blue sand", + "plastic hourglass with glass center", + "compact sand clock with glass middle", + "blue hourglass with white frame ends", + "smooth hourglass with cylindrical frame", + "hourglass shaped timer with smooth finish", + "glass hourglass supported by plastic frame" + ], + "unseen": [ + "sand clock with top and bottom caps", + "sand clock with two conical chambers", + "sand clock with blue and white chambers" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/046_alarm-clock/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/046_alarm-clock/base0.json new file mode 100644 index 00000000..104d1d60 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/046_alarm-clock/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "alarm clock", + "seen": [ + "black alarm clock", + "digital alarm clock", + "time and alarm clock", + "green digit alarm clock", + "medium-sized alarm clock", + "rectangular black alarm clock", + "alarm clock with FM indicator", + "alarm clock with clear screen", + "black clock with sleek design", + "plastic alarm clock with glass screen", + "black clock with green glowing numbers", + "desktop alarm clock with curved design" + ], + "unseen": [ + "smooth black alarm clock", + "curved-bottom alarm clock", + "alarm clock with green display" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/046_alarm-clock/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/046_alarm-clock/base1.json new file mode 100644 index 00000000..850e142f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/046_alarm-clock/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "alarm-clock", + "seen": [ + "black alarm-clock", + "small alarm-clock", + "compact alarm-clock", + "palm-sized alarm-clock", + "rectangular alarm-clock", + "black matte alarm-clock", + "smooth black alarm-clock", + "digital screen alarm-clock", + "alarm-clock with rounded edges", + "lightweight plastic alarm-clock", + "alarm-clock with buttons on side", + "alarm-clock with digital display" + ], + "unseen": [ + "alarm-clock with beige screen", + "alarm-clock for bedside table", + "alarm-clock with large numbers" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/046_alarm-clock/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/046_alarm-clock/base2.json new file mode 100644 index 00000000..3f632b5b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/046_alarm-clock/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "alarm-clock", + "seen": [ + "orange clock face", + "clock with slanted sides", + "smooth shiny alarm-clock", + "date and time alarm-clock", + "modern styled alarm-clock", + "alarm-clock with black border", + "small clock with date display", + "orange face with white numbers", + "black and white plastic casing", + "compact rectangular alarm-clock", + "digital date feature alarm-clock", + "palm-sized alarm-clock, bright orange" + ], + "unseen": [ + "alarm-clock", + "orange face black edges clock", + "rectangular clock with orange dial" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/046_alarm-clock/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/046_alarm-clock/base3.json new file mode 100644 index 00000000..3b8e6294 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/046_alarm-clock/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "alarm clock", + "seen": [ + "dark blue clock", + "square alarm clock", + "clock with white dial", + "clock with black hands", + "white-faced alarm clock", + "small plastic alarm clock", + "palm-sized dark blue clock", + "square plastic alarm clock", + "blue clock with green stripe", + "alarm clock with green stripe", + "small clock with rounded edges", + "rounded square-shaped alarm clock" + ], + "unseen": [ + "blue alarm clock", + "compact alarm clock", + "alarm clock with smooth finish" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/046_alarm-clock/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/046_alarm-clock/base4.json new file mode 100644 index 00000000..2dde0604 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/046_alarm-clock/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "alarm-clock", + "seen": [ + "dark alarm-clock", + "desk alarm-clock", + "brown alarm-clock", + "curvy alarm-clock", + "rounded alarm-clock", + "medium-sized alarm-clock", + "brown curved alarm-clock", + "alarm-clock with curved edges", + "alarm-clock with palm-fit size", + "alarm-clock with smooth finish", + "alarm-clock for time and alarms", + "alarm-clock with circular clock face" + ], + "unseen": [ + "plastic alarm-clock", + "compact alarm-clock", + "smooth dark brown alarm-clock" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/046_alarm-clock/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/046_alarm-clock/base5.json new file mode 100644 index 00000000..5f825e41 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/046_alarm-clock/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "alarm clock", + "seen": [ + "brown clock", + "rectangular brown alarm clock", + "clock with light-colored face", + "clock with black time markers", + "rounded-edge brown alarm clock", + "smooth brown plastic alarm clock", + "brown clock with light dial face", + "plastic alarm clock with time dial", + "clock with black hands and numbers", + "rectangular brown timekeeping clock", + "medium-sized clock with alarm feature", + "rectangular clock with alarm function" + ], + "unseen": [ + "alarm clock", + "medium rectangular alarm clock", + "dark brown clock with rounded corners" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/047_mouse/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/047_mouse/base0.json new file mode 100644 index 00000000..b5d0979d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/047_mouse/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "mouse", + "seen": [ + "plastic mouse", + "dark gray mouse", + "oval-shaped mouse", + "mouse with curved top", + "rounded computer mouse", + "mouse with scroll wheel", + "dark gray palm-sized mouse", + "mouse with ergonomic design", + "mouse with buttons and wheel", + "smooth dark gray computer mouse", + "wireless dark gray computer mouse", + "dark gray mouse with smooth texture" + ], + "unseen": [ + "hand-sized dark gray mouse", + "small rounded plastic mouse", + "dark gray computer mouse with logo" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/047_mouse/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/047_mouse/base1.json new file mode 100644 index 00000000..32b4e6aa --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/047_mouse/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "mouse", + "seen": [ + "dark gray mouse", + "ergonomic mouse", + "rounded gray mouse", + "palm-fitting mouse", + "small computer mouse", + "smooth plastic mouse", + "mouse with sleek design", + "mouse for cursor control", + "compact oval-shaped mouse", + "dark mouse with logo on top", + "gray mouse with smooth surface", + "mouse with two buttons and wheel" + ], + "unseen": [ + "flat-bottomed mouse", + "mouse with scroll wheel", + "hand-sized wireless mouse" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/047_mouse/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/047_mouse/base2.json new file mode 100644 index 00000000..c0c2850b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/047_mouse/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "mouse", + "seen": [ + "gray mouse", + "computer mouse", + "small mouse with smooth surface", + "plastic mouse with smooth finish", + "mouse with curved ergonomic shape", + "compact matte gray computer mouse", + "mouse with scroll wheel in center", + "rounded top mouse with glossy logo", + "oval-shaped mouse for computer use", + "dark mouse with slight textured sides", + "mouse with two buttons and a scroll wheel", + "dark ergonomic mouse for computer navigation" + ], + "unseen": [ + "dark gray wireless mouse", + "hand-sized black-gray mouse", + "wireless mouse with left and right buttons" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/048_stapler/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/048_stapler/base0.json new file mode 100644 index 00000000..4b0474fa --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/048_stapler/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "stapler", + "seen": [ + "blue stapler", + "manual blue stapler", + "stapler with curved blue top", + "curved blue and silver stapler", + "stapler with smooth blue finish", + "metallic stapler with blue cover", + "handheld blue and silver stapler", + "blue stapler with ergonomic curve", + "plastic stapler with metal on base", + "compact stapler with smooth surface", + "blue stapler with shiny silver parts", + "blue plastic stapler with silver metal" + ], + "unseen": [ + "small hand-sized stapler", + "blue stapler for holding papers", + "small stapler with simple design" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/048_stapler/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/048_stapler/base1.json new file mode 100644 index 00000000..f3ab394c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/048_stapler/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "stapler", + "seen": [ + "curved black stapler", + "compact black stapler", + "handheld black stapler", + "black stapler with curved top", + "black smooth-textured stapler", + "stapler with silver staple tray", + "black stapler for joining pages", + "black stapler with smooth finish", + "stapler with silver inner plates", + "black stapler with shiny interior", + "black stapler with rectangular base", + "small black stapler with metal core" + ], + "unseen": [ + "black stapler", + "palm-sized black stapler", + "metal and plastic stapler" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/048_stapler/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/048_stapler/base2.json new file mode 100644 index 00000000..eb455361 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/048_stapler/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "stapler", + "seen": [ + "blue stapler", + "small stapler", + "handheld stapler", + "compact blue stapler", + "metal and plastic stapler", + "stapler for stapling paper", + "stapler with black top section", + "small lightweight blue stapler", + "blue stapler with curved design", + "blue stapler with black handle area", + "plastic blue stapler with metal parts", + "stapler with white metallic components" + ], + "unseen": [ + "plastic stapler", + "blue stapler with smooth finish", + "stapler with white loading section" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/048_stapler/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/048_stapler/base3.json new file mode 100644 index 00000000..0b55851d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/048_stapler/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "stapler", + "seen": [ + "handheld stapler", + "small office stapler", + "modern design stapler", + "smooth plastic stapler", + "plastic and metal stapler", + "stapler with orange button", + "stapler with rounded edges", + "stapler for fastening papers", + "stapler with orange top lever", + "paper stapler with silver tray", + "white body stapler with accents", + "white stapler with rounded design" + ], + "unseen": [ + "white stapler", + "compact stapler", + "white stapler with metal tray" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/048_stapler/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/048_stapler/base4.json new file mode 100644 index 00000000..b89fab9b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/048_stapler/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "stapler", + "seen": [ + "red stapler", + "stapler with red top", + "small handheld stapler", + "compact office stapler", + "desk stapler with red body", + "red stapler with black base", + "red top black bottom stapler", + "stapler for fastening papers", + "manual stapler with black base", + "stapler with metallic staple slot", + "red plastic stapler with smooth top", + "office stapler with plastic and metal parts" + ], + "unseen": [ + "red plastic stapler", + "red stapler with silver chamber", + "paper stapler with curved handle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/048_stapler/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/048_stapler/base5.json new file mode 100644 index 00000000..1d775e54 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/048_stapler/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "stapler", + "seen": [ + "red stapler", + "compact stapler", + "small red stapler", + "curved red stapler", + "handheld red stapler", + "lightweight red stapler", + "stapler in bright red color", + "metallic stapler with red body", + "plastic stapler with metal top", + "red stapler for binding papers", + "compact red stapler with silver accents", + "stapler with rectangular silver mechanism" + ], + "unseen": [ + "shiny red plastic stapler", + "stapler with silver metal top", + "stapler with smooth plastic body" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/048_stapler/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/048_stapler/base6.json new file mode 100644 index 00000000..008e54ae --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/048_stapler/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "stapler", + "seen": [ + "blue stapler", + "simple stapler", + "bright blue smooth stapler", + "compact bright blue stapler", + "blue stapler with white accents", + "stapler with blue plastic handle", + "blue stapler for paper fastening", + "handy blue stapler with metal parts", + "blue rectangular stapler with curves", + "blue stapler with silver staple tray", + "plastic stapler with silver mechanism", + "plastic stapler with metallic interior" + ], + "unseen": [ + "blue and white stapler", + "hand-size bright blue stapler", + "rectangular stapler with curved corners" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/049_shampoo/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/049_shampoo/base1.json new file mode 100644 index 00000000..022852f4 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/049_shampoo/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "shampoo", + "seen": [ + "white shampoo bottle", + "curved shampoo bottle", + "plastic shampoo bottle", + "blue top shampoo bottle", + "handheld shampoo container", + "bottle with smooth surface", + "shampoo bottle with blue cap", + "bottle with hexagonal blue cap", + "shampoo bottle with curved sides", + "white and blue shampoo container", + "plastic shampoo with orange label", + "medium-sized white plastic shampoo" + ], + "unseen": [ + "blue-capped shampoo bottle", + "white shampoo with orange design", + "shampoo container in white and blue" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/049_shampoo/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/049_shampoo/base2.json new file mode 100644 index 00000000..6ee05068 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/049_shampoo/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "shampoo", + "seen": [ + "shampoo bottle", + "white shampoo bottle", + "bottle with yellow cap", + "medium shampoo container", + "hand-held shampoo bottle", + "smooth shampoo container", + "white bottle with golden accents", + "white shampoo bottle with golden top", + "shampoo bottle with rectangular base", + "bottle for shampoo with glossy finish", + "plastic shampoo bottle with smooth surface", + "rectangular shampoo bottle with rounded top" + ], + "unseen": [ + "medium-sized shampoo bottle", + "white plastic shampoo bottle", + "shampoo bottle with rounded yellow cap" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/049_shampoo/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/049_shampoo/base3.json new file mode 100644 index 00000000..e5a3853e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/049_shampoo/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "shampoo", + "seen": [ + "pink shampoo bottle", + "pink bottle with narrow top", + "medium pink bottle for shampoo", + "pink bottle with black dispenser", + "hand-sized pink shampoo container", + "rectangular pink shampoo container", + "smooth pink plastic shampoo bottle", + "shampoo bottle with shiny pink body", + "pink shampoo container with pump top", + "shampoo bottle with rectangular shape", + "plastic shampoo bottle with black pump", + "rectangular pink bottle with dispenser" + ], + "unseen": [ + "glossy pink shampoo bottle", + "plastic pink shampoo bottle", + "shampoo bottle with black pump" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/049_shampoo/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/049_shampoo/base4.json new file mode 100644 index 00000000..cea6caee --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/049_shampoo/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "shampoo", + "seen": [ + "white shampoo bottle", + "medium shampoo bottle", + "plastic shampoo container", + "hand-sized shampoo bottle", + "oval-shaped shampoo bottle", + "pink-topped shampoo bottle", + "glossy white shampoo bottle", + "shampoo bottle with pink cap", + "smooth white shampoo dispenser", + "white body pink flip cap bottle", + "shampoo dispenser with flat base", + "medium oval plastic shampoo container" + ], + "unseen": [ + "Dove shampoo bottle", + "bottle for holding shampoo", + "shampoo bottle with Dove logo" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/049_shampoo/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/049_shampoo/base5.json new file mode 100644 index 00000000..826b753d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/049_shampoo/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "shampoo", + "seen": [ + "yellow shampoo bottle", + "bottle with cartoon label", + "yellow bottle with green top", + "smooth bottle with green top", + "medium shampoo bottle with pump", + "shampoo bottle with handle slot", + "hand-sized yellow shampoo bottle", + "yellow container with green pump", + "shampoo bottle with rounded body", + "plastic shampoo bottle with label", + "shampoo with green dispensing pump", + "yellow shampoo with cartoon design" + ], + "unseen": [ + "smooth plastic shampoo container", + "yellow bottle with rounded edges", + "yellow shampoo with pump dispenser" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/049_shampoo/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/049_shampoo/base6.json new file mode 100644 index 00000000..8251878f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/049_shampoo/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "shampoo", + "seen": [ + "bottle of shampoo", + "blue shampoo bottle", + "hand-sized shampoo bottle", + "small smooth shampoo bottle", + "shampoo bottle with white cap", + "bottle light blue rounded shape", + "shampoo bottle flat bottom edge", + "curved light blue shampoo bottle", + "shampoo bottle with matte plastic", + "shampoo bottle tapered toward cap", + "blue shampoo bottle slender design", + "light blue shampoo bottle small size" + ], + "unseen": [ + "smooth bottle containing shampoo", + "oval light blue shampoo container", + "shampoo container white rounded top" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/049_shampoo/base7.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/049_shampoo/base7.json new file mode 100644 index 00000000..cb883bfc --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/049_shampoo/base7.json @@ -0,0 +1,22 @@ +{ + "raw_description": "shampoo", + "seen": [ + "green shampoo bottle", + "plastic shampoo container", + "hand-sized shampoo bottle", + "medium size shampoo bottle", + "dark green bottle with pump", + "shampoo bottle with black pump", + "shampoo bottle with smooth surface", + "shampoo bottle with matte black pump", + "shampoo bottle, green with black cap", + "rectangular dark green shampoo bottle", + "shampoo container with dispensing pump", + "rectangular green plastic shampoo dispenser" + ], + "unseen": [ + "green bottle with pump top", + "dark green pump shampoo bottle", + "shampoo bottle for personal use" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/050_bell/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/050_bell/base0.json new file mode 100644 index 00000000..47801a7c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/050_bell/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bell", + "seen": [ + "white bell", + "white dome bell", + "compact tabletop bell", + "plastic and metal bell", + "bell with black flat base", + "small desk bell for tapping", + "white bell with black bottom", + "white and black colored bell", + "rounded bell with flat black part", + "bell shaped like half-circle dome", + "simple hand bell with black stand", + "bell with metallic top and plastic base" + ], + "unseen": [ + "medium-sized bell", + "smooth metal bell", + "lightweight bell with smooth surface" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/050_bell/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/050_bell/base1.json new file mode 100644 index 00000000..e3771c28 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/050_bell/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bell", + "seen": [ + "blue bell", + "round bell", + "smooth blue bell", + "flat-topped bell", + "brown-bottomed bell", + "bell with brown base", + "blue dome-shaped bell", + "bell with circular body", + "small blue and brown bell", + "bell with protruding knob", + "compact bell with round base", + "metal bell with plastic parts" + ], + "unseen": [ + "palm-sized bell", + "bell with raised round top", + "blue bell with glossy finish" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/051_candlestick/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/051_candlestick/base0.json new file mode 100644 index 00000000..44ca1738 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/051_candlestick/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "candlestick", + "seen": [ + "small candlestick", + "blue-green candlestick", + "hand-sized candlestick", + "flower base candlestick", + "rounded stem candlestick", + "flower-shaped candlestick", + "gold-decorated candlestick", + "smooth textured candlestick", + "small flower-like base candlestick", + "candlestick with smooth ceramic body", + "decorative candlestick with gold patterns", + "blue-green candlestick with golden accents" + ], + "unseen": [ + "ceramic candle holder", + "candlestick with curved stem and flower base", + "blue-green ceramic candlestick shaped like flower" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/051_candlestick/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/051_candlestick/base1.json new file mode 100644 index 00000000..113eafbe --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/051_candlestick/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "candlestick", + "seen": [ + "dark candlestick", + "brown candlestick", + "polished candlestick", + "wooden or metal candlestick", + "medium-sized dark candlestick", + "carved dark brown candlestick", + "rectangular-based candlestick", + "decorative carved candlestick", + "candlestick with carved surface", + "candlestick for holding candles", + "smooth brown candlestick column", + "candlestick with rectangular base" + ], + "unseen": [ + "smooth candlestick", + "dark brown candle holder", + "medium candlestick with smooth texture" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/051_candlestick/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/051_candlestick/base2.json new file mode 100644 index 00000000..735be354 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/051_candlestick/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "candlestick", + "seen": [ + "metal candlestick", + "golden candlestick", + "golden candle holder", + "smooth gold candlestick", + "seven-arm candle holder", + "curved metal candlestick", + "candlestick with seven arms", + "branched golden candlestick", + "candlestick with curved arms", + "metallic branched candlestick", + "gold candlestick with curved branches", + "golden candlestick with smooth surface" + ], + "unseen": [ + "medium gold candle holder", + "medium-sized candle holder", + "smooth metallic candlestick" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/051_candlestick/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/051_candlestick/base3.json new file mode 100644 index 00000000..83055b4b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/051_candlestick/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "candlestick", + "seen": [ + "golden candlestick", + "metallic candlestick", + "medium-sized candlestick", + "smooth golden candlestick", + "candlestick with wide round base", + "candle holder with detailed design", + "tall candlestick with multiple arms", + "ornate candlestick with curved arms", + "candlestick with curved candle holders", + "candlestick with shiny metallic finish", + "candlestick with multiple candle slots", + "decorative candlestick for holding candles" + ], + "unseen": [ + "classic metallic candlestick", + "candlestick with symmetrical branches", + "golden candlestick with circular platform" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/051_candlestick/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/051_candlestick/base4.json new file mode 100644 index 00000000..5867346f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/051_candlestick/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "candlestick", + "seen": [ + "bronze candlestick", + "three-arm candlestick", + "candlestick with curved arms", + "three-holder bronze candlestick", + "medium-sized bronze candleholder", + "metal candlestick with smooth texture", + "candlestick with polished smooth finish", + "three-arm candleholder with bronze sheen", + "bronze tabletop candlestick with holders", + "smooth bronze candlestick with round base", + "three-armed candleholder with curved design", + "candleholder with bronze finish and round base" + ], + "unseen": [ + "bronze stand for candles", + "metal candleholder with circular base", + "metallic bronze candlestick for holding candles" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/052_dumbbell/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/052_dumbbell/base0.json new file mode 100644 index 00000000..b82a1ad4 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/052_dumbbell/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "dumbbell", + "seen": [ + "black dumbbell", + "handheld dumbbell", + "gray handle dumbbell", + "black plate dumbbell", + "black and gray dumbbell", + "cylindrical dumbbell bar", + "rubber and steel dumbbell", + "compact dumbbell for lifting", + "gym dumbbell with smooth grip", + "textured black dumbbell weights", + "smooth dumbbell for gym training", + "medium dumbbell with black weights" + ], + "unseen": [ + "round weight dumbbell", + "dumbbell with round plates", + "metal dumbbell with rubber plates" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/052_dumbbell/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/052_dumbbell/base1.json new file mode 100644 index 00000000..13cdf786 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/052_dumbbell/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "dumbbell", + "seen": [ + "matte red dumbbell", + "hex weight dumbbell", + "hexagonal red dumbbell", + "bright red gym dumbbell", + "solid matte red dumbbell", + "hex red workout dumbbell", + "single bright red dumbbell", + "bright red handheld dumbbell", + "exercise dumbbell with hex ends", + "red dumbbell with straight handle", + "red dumbbell with hexagonal weights", + "medium dumbbell for weight training" + ], + "unseen": [ + "red dumbbell", + "medium-sized red dumbbell", + "hexagonal dumbbell with smooth finish" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/052_dumbbell/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/052_dumbbell/base2.json new file mode 100644 index 00000000..918d5ac4 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/052_dumbbell/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "dumbbell", + "seen": [ + "hex dumbbell", + "black dumbbell", + "gray and black dumbbell", + "black handheld dumbbell", + "rubber-coated hex dumbbell", + "compact dumbbell for exercise", + "dumbbell with hexagonal weights", + "dumbbell with matte black finish", + "medium-sized dumbbell for lifting", + "hexagonal dumbbell with smooth bar", + "black dumbbell for lifting weights", + "hex dumbbell with visible weight numbers" + ], + "unseen": [ + "metal bar dumbbell", + "gym weight dumbbell", + "dumbbell with rubber-coated grips" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/052_dumbbell/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/052_dumbbell/base3.json new file mode 100644 index 00000000..6c9a0e03 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/052_dumbbell/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "dumbbell", + "seen": [ + "gym dumbbell", + "black dumbbell", + "rubber-coated black dumbbell", + "black dumbbell for weightlifting", + "dumbbell with textured white grip", + "medium dumbbell with black weights", + "black dumbbell with white grip bar", + "black dumbbell for muscle exercises", + "dumbbell with smooth black cylinders", + "black dumbbell with thick center bar", + "black dumbbell two cylindrical weights", + "dumbbell with cylindrical black weights" + ], + "unseen": [ + "round weights dumbbell", + "black dumbbell with white metal handle", + "compact dumbbell with black rubber weights" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/052_dumbbell/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/052_dumbbell/base4.json new file mode 100644 index 00000000..1cd3d6a4 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/052_dumbbell/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "dumbbell", + "seen": [ + "workout dumbbell", + "hexagonal dumbbell", + "black hex dumbbell", + "black dumbbell with silver handle", + "dumbbell with black rubber weights", + "hexagonal black weights and shiny bar", + "black dumbbell with central metal bar", + "medium weight dumbbell with rubber ends", + "gym dumbbell with rubber-coated weights", + "hexagonal black dumbbell with silver grip", + "hand-sized dumbbell with hexagonal weights", + "dumbbell with shiny handle and rubber ends" + ], + "unseen": [ + "black dumbbell", + "strength training dumbbell", + "black hexagonal dumbbell for workouts" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/052_dumbbell/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/052_dumbbell/base5.json new file mode 100644 index 00000000..80762e3c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/052_dumbbell/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "dumbbell", + "seen": [ + "blue dumbbell", + "solid blue dumbbell", + "compact gym dumbbell", + "rubber-coated dumbbell", + "exercise weight dumbbell", + "rounded-end blue dumbbell", + "single-piece blue dumbbell", + "blue dumbbell for workouts", + "smooth-coated blue dumbbell", + "blue dumbbell with rounded ends", + "blue dumbbell with smooth finish", + "blue dumbbell with cylindrical grip" + ], + "unseen": [ + "hand-sized blue dumbbell", + "blue dumbbell for lifting", + "small hand-held weight dumbbell" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/052_dumbbell/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/052_dumbbell/base6.json new file mode 100644 index 00000000..7a984f46 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/052_dumbbell/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "dumbbell", + "seen": [ + "purple dumbbell", + "exercise dumbbell", + "hexagonal dumbbell", + "solid purple dumbbell", + "rubber-coated dumbbell", + "single weight dumbbell", + "grippy purple dumbbell", + "handheld purple dumbbell", + "purple dumbbell for lifting", + "medium purple weight dumbbell", + "purple dumbbell with hexagonal ends", + "medium-sized dumbbell with hexagon ends" + ], + "unseen": [ + "smooth purple dumbbell", + "fitness purple dumbbell", + "dumbbell with straight connecting bar" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/053_teanet/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/053_teanet/base1.json new file mode 100644 index 00000000..877af306 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/053_teanet/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "teanet", + "seen": [ + "smooth teanet", + "ceramic teanet", + "teanet with handle", + "medium-sized teanet", + "teanet with round base", + "teanet with long brown handle", + "ceramic teanet for brewing tea", + "teanet with brown wooden handle", + "white teanet with brown accents", + "handheld teanet with wooden grip", + "white teanet with engraved patterns", + "engraved teanet with floral patterns" + ], + "unseen": [ + "white teanet", + "cone-shaped teanet", + "teanet with cone-like body" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/053_teanet/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/053_teanet/base4.json new file mode 100644 index 00000000..ded17edd --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/053_teanet/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "teanet", + "seen": [ + "metal teanet", + "round teanet", + "metal mesh teanet", + "tea filter teanet", + "hand-sized teanet", + "medium-sized teanet", + "smooth-handled teanet", + "teanet with fine mesh", + "teanet with attached hook", + "teanet with rounded bowl shape", + "light gray teanet with plastic handle", + "light gray mesh teanet with beige handle" + ], + "unseen": [ + "gray teanet", + "teanet with beige handle", + "teanet for filtering tea leaves" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/053_teanet/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/053_teanet/base5.json new file mode 100644 index 00000000..810c3d5a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/053_teanet/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "teanet", + "seen": [ + "metal teanet", + "small tea strainer", + "small round teanet", + "round silver teanet", + "silver cup strainer", + "teanet with tiny holes", + "perforated metal teanet", + "hand-sized tea strainer", + "silver teanet with holes", + "teanet with perforations", + "metallic teanet with handles", + "tea strainer with long handle" + ], + "unseen": [ + "silver teanet", + "cup-shaped tea strainer", + "metal teanet with handle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/053_teanet/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/053_teanet/base6.json new file mode 100644 index 00000000..eec52552 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/053_teanet/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "teanet", + "seen": [ + "teanet", + "small teanet", + "silver teanet", + "teanet for tea", + "semi-sphere teanet", + "brown-handled teanet", + "teanet with brown handle", + "silver teanet with holes", + "teanet with curved mesh body", + "small teanet for brewing tea", + "teanet with silver mesh sphere", + "metal teanet with straight handle" + ], + "unseen": [ + "palm-sized teanet", + "smooth teanet with mesh", + "teanet with perforated mesh" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/053_teanet/base7.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/053_teanet/base7.json new file mode 100644 index 00000000..74cf584c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/053_teanet/base7.json @@ -0,0 +1,22 @@ +{ + "raw_description": "teanet", + "seen": [ + "white teanet", + "tea strainer", + "small bowl-shaped teanet", + "teanet with black plastic handle", + "metal teanet with perforated holes", + "white bowl-shaped teanet for straining", + "compact teanet made of metal and plastic", + "metal teanet with smooth polished surface", + "lightweight teanet with smooth black handle", + "medium-sized teanet with perforated surface", + "semi-spherical teanet with perforated walls", + "semi-spherical tea strainer with black handle" + ], + "unseen": [ + "simple white and black teanet", + "white teanet with long black handle", + "handheld teanet for liquid straining" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/054_baguette/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/054_baguette/base2.json new file mode 100644 index 00000000..33c8f235 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/054_baguette/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "baguette", + "seen": [ + "golden brown baguette", + "baguette with oval shape", + "forearm-sized oval bread", + "baguette bread for eating", + "golden baguette for slicing", + "crusty long bread with ridges", + "brown baguette with oval form", + "medium-sized brown baguette loaf", + "crusty bread loaf called baguette", + "ridged golden crusty baguette bread", + "medium forearm-sized crusty baguette", + "golden brown oblong-shaped bread loaf" + ], + "unseen": [ + "loaf with ridged crust", + "long bread with ridged top", + "long oval bread loaf baguette" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/054_baguette/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/054_baguette/base3.json new file mode 100644 index 00000000..0bb312a0 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/054_baguette/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "baguette", + "seen": [ + "oval-shaped baguette", + "crispy baguette loaf", + "golden brown baguette", + "freshly baked baguette", + "soft-centered baguette", + "long loaf bread baguette", + "rectangular loaf baguette", + "medium-sized baguette loaf", + "light golden baguette bread", + "stretched oval bread baguette", + "baguette loaf with crusty texture", + "baguette with crusty ridged surface" + ], + "unseen": [ + "baguette with golden crust", + "baguette with smooth sides", + "golden baguette with ridges" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/055_small-speaker/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/055_small-speaker/base1.json new file mode 100644 index 00000000..8c5271f4 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/055_small-speaker/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "small speaker", + "seen": [ + "black speaker", + "glossy speaker", + "red and black speaker", + "handheld small speaker", + "speaker with red base color", + "red back black front speaker", + "angled glossy plastic speaker", + "small speaker with shiny finish", + "rectangular black-and-red speaker", + "black front red back compact speaker", + "mini rectangular glossy black speaker", + "portable small speaker with black front" + ], + "unseen": [ + "compact speaker", + "slanted box-shaped speaker", + "angled small handheld speaker" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/055_small-speaker/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/055_small-speaker/base2.json new file mode 100644 index 00000000..461ea93b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/055_small-speaker/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "small speaker", + "seen": [ + "black round speaker", + "small round speaker", + "spherical small speaker", + "hand-sized black speaker", + "mesh-covered small speaker", + "speaker covered in black mesh", + "small speaker for sound output", + "compact spherical audio speaker", + "small speaker with woven texture", + "black speaker with mesh material", + "portable black spherical speaker", + "fabric-textured small black speaker" + ], + "unseen": [ + "black small speaker", + "spherical black sound speaker", + "small speaker with fabric mesh" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/055_small-speaker/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/055_small-speaker/base3.json new file mode 100644 index 00000000..b2820991 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/055_small-speaker/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "small speaker", + "seen": [ + "small speaker", + "tiny sound box", + "compact rectangular speaker", + "speaker with perforated top", + "smooth dark gray audio speaker", + "dark gray plastic sound device", + "compact speaker with side vents", + "tiny speaker with rounded edges", + "portable palm-sized sound speaker", + "dark gray speaker with plastic body", + "small speaker with top perforations", + "dark gray speaker with smooth finish" + ], + "unseen": [ + "dark gray small speaker", + "rounded rectangular dark gray speaker", + "palm-sized speaker with rounded corners" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/056_switch/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/056_switch/base0.json new file mode 100644 index 00000000..1fbdfe89 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/056_switch/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "switch", + "seen": [ + "black switch", + "small switch", + "red and black switch", + "red-top black switch", + "switch with curved red toggle", + "rectangular black plastic switch", + "black base switch with red toggle", + "compact switch with toggle on top", + "plastic switch with metal terminals", + "rectangular switch with curved lever", + "black switch with metallic connectors", + "hand-sized switch with connector prongs" + ], + "unseen": [ + "simple toggle switch", + "switch with two metal prongs", + "smooth black switch with red lever" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/056_switch/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/056_switch/base1.json new file mode 100644 index 00000000..285256ae --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/056_switch/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "switch", + "seen": [ + "beige switch", + "light plastic switch", + "small trapezoid switch", + "switch with brown button", + "switch with a smooth top", + "hand-sized light beige switch", + "tiny matte beige toggle switch", + "beige switch with smooth finish", + "matte beige switch with flat top", + "switch with rectangular dark button", + "toggle switch with brown rectangle button", + "light-colored trapezoid-shaped plastic switch" + ], + "unseen": [ + "beige switch with darker brown top", + "small beige switch with angular sides", + "rectangular button switch on beige base" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/056_switch/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/056_switch/base2.json new file mode 100644 index 00000000..4f70dfc8 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/056_switch/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "switch", + "seen": [ + "tan switch", + "flat switch", + "small light tan switch", + "switch with darker rectangle", + "angled switch light tan color", + "plastic switch with angled edges", + "hand-sized switch tan smooth finish", + "switch trapezoid shape brown marking", + "rectangular switch with slanted sides", + "smooth tan switch with textured sides", + "trapezoid-shaped switch smooth surface", + "light tan rectangular switch smooth finish" + ], + "unseen": [ + "small flat switch tan top", + "switch flat tan top brown rectangle", + "lightweight plastic switch angled edges" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/056_switch/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/056_switch/base3.json new file mode 100644 index 00000000..62ac3c86 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/056_switch/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "switch", + "seen": [ + "black switch", + "control switch", + "palm-sized black switch", + "switch with slanted sides", + "black switch with slim profile", + "sleek black rectangular switch", + "black switch with matte coating", + "black rectangular control switch", + "compact rectangular black switch", + "slanted-edge plastic black switch", + "medium-sized black rectangular switch", + "rectangular switch with smooth surface" + ], + "unseen": [ + "smooth black switch", + "rectangular black switch", + "matte black plastic switch" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/056_switch/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/056_switch/base4.json new file mode 100644 index 00000000..7292f88d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/056_switch/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "switch", + "seen": [ + "black base switch", + "rectangular switch", + "tiny rectangular switch", + "two-tone red black switch", + "metal-pronged black switch", + "small switch with red lever", + "switch with three metal pins", + "smooth red switch with labels", + "red and black electrical switch", + "plastic switch with metal prongs", + "toggle switch with black housing", + "switch for turning circuits on off" + ], + "unseen": [ + "red toggle switch", + "curved toggle switch", + "red ON OFF labeled toggle switch" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/056_switch/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/056_switch/base5.json new file mode 100644 index 00000000..020ee368 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/056_switch/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "switch", + "seen": [ + "smooth gray switch", + "small square switch", + "square gray switch panel", + "light gray plastic switch", + "palm-sized light gray switch", + "switch for electrical control", + "flat square with raised center", + "angled-edge gray square switch", + "square switch with smooth surface", + "small square switch made of plastic", + "switch with rectangular raised area", + "switch with smooth light gray finish" + ], + "unseen": [ + "flat gray switch", + "switch with angled edges", + "gray switch with slanted sides" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/056_switch/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/056_switch/base6.json new file mode 100644 index 00000000..18bbec00 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/056_switch/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "switch", + "seen": [ + "switch", + "red bar switch", + "light gray square switch", + "plastic gray body switch", + "square switch with red bar", + "gray switch with red toggle", + "switch with thin red toggle", + "control switch with red lever", + "compact switch with flat base", + "toggle switch light gray base", + "switch for electrical control", + "electrical switch with protruding toggle" + ], + "unseen": [ + "flat square gray switch", + "smooth gray base switch", + "hand-sized square electrical switch" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/056_switch/base7.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/056_switch/base7.json new file mode 100644 index 00000000..15a1f2c5 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/056_switch/base7.json @@ -0,0 +1,22 @@ +{ + "raw_description": "switch", + "seen": [ + "handheld switch", + "smooth gray switch", + "gray plastic switch", + "electric toggle switch", + "two-button gray switch", + "trapezoidal base switch", + "gray smooth surface switch", + "gray base with black buttons", + "switch with trapezoidal base", + "gray switch with black buttons", + "plastic switch with matte buttons", + "medium switch with trapezoidal base" + ], + "unseen": [ + "gray switch", + "gray switch medium size", + "switch with circular black buttons" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/057_toycar/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/057_toycar/base0.json new file mode 100644 index 00000000..862a8911 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/057_toycar/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "toycar", + "seen": [ + "small toycar", + "plastic toycar", + "toycar for kids", + "glossy blue toycar", + "lightweight toycar", + "bright blue toycar", + "toycar with rounded edges", + "palm-sized plastic toycar", + "small blue car-shaped toycar", + "blue toycar with white windows", + "toycar with smiling face design", + "toycar with black and orange wheels" + ], + "unseen": [ + "blue toycar", + "toycar with orange wheels", + "toycar with smooth finish" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/057_toycar/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/057_toycar/base1.json new file mode 100644 index 00000000..51c2f268 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/057_toycar/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "toycar", + "seen": [ + "blue toycar", + "blue top toycar", + "bright sleek toycar", + "small plastic toycar", + "blue and gray toycar", + "toycar for kids play", + "palm-sized blue toycar", + "toycar with black tires", + "toycar with shiny wheels", + "toycar with smooth texture", + "toycar with shiny plastic finish", + "toycar with transparent windshield" + ], + "unseen": [ + "gray base toycar", + "toycar with rounded body", + "compact toycar with rotating wheels" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/057_toycar/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/057_toycar/base2.json new file mode 100644 index 00000000..aa1088e9 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/057_toycar/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "toycar", + "seen": [ + "pink toycar", + "small pink plastic toycar", + "small toycar for kids play", + "pink car with purple windows", + "toycar made from smooth plastic", + "pink toycar with rounded corners", + "pink toycar shaped like real car", + "toycar with rounded compact shape", + "toycar with purple tinted windows", + "small pink car with smooth texture", + "miniature pink car with black wheels", + "hand-sized toycar with white roof bars" + ], + "unseen": [ + "pink toycar with white bumper", + "tiny pink car with four wheels", + "compact toycar with shiny black tires" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/057_toycar/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/057_toycar/base3.json new file mode 100644 index 00000000..7b16d9af --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/057_toycar/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "toycar", + "seen": [ + "green toycar", + "tiny green toycar", + "small green toycar", + "shiny green toycar", + "smooth green toycar", + "playful green toycar", + "green car-shaped toycar", + "toycar with black wheels", + "small plastic green toycar", + "miniature bright green toycar", + "green toycar made from plastic", + "toycar with black roof and wheels" + ], + "unseen": [ + "plastic toycar", + "bright green toycar", + "toycar with rounded edges" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/057_toycar/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/057_toycar/base4.json new file mode 100644 index 00000000..2c149224 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/057_toycar/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "toycar", + "seen": [ + "small plastic toycar", + "toycar with black wheels", + "hand-sized rolling toycar", + "toycar with rounded edges", + "compact glossy blue toycar", + "kids' toycar in bright blue", + "toycar with dark gray windows", + "toycar made of plastic material", + "plastic toycar with smooth finish", + "bright blue toycar with roof buttons", + "blue toycar with yellow roof accents", + "toycar featuring black and white wheels" + ], + "unseen": [ + "blue toycar", + "blue toycar with yellow buttons", + "playable toycar with rolling wheels" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/057_toycar/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/057_toycar/base5.json new file mode 100644 index 00000000..f284af19 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/057_toycar/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "toycar", + "seen": [ + "mini car toy", + "small car toy", + "smooth teal car", + "compact teal toycar", + "rounded teal car toy", + "toycar with black tires", + "toycar with glossy finish", + "play car in teal and white", + "miniature teal vehicle toy", + "light teal toycar with white roof", + "small teal plastic car with windows", + "toycar with white roof and teal body" + ], + "unseen": [ + "teal toycar", + "plastic teal car with wheels", + "plastic toycar with four spinning wheels" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/058_markpen/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/058_markpen/base0.json new file mode 100644 index 00000000..2a3c3e2a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/058_markpen/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "markpen", + "seen": [ + "black markpen", + "slim pen for marking", + "marker with tube-like shape", + "slim black and white markpen", + "markpen with black color tip", + "black marker with rounded tip", + "black markpen with small size", + "smooth-texture plastic markpen", + "handheld black and white marker", + "writing markpen black and white", + "markpen with white middle section", + "black marker with white tube body" + ], + "unseen": [ + "plastic markpen", + "markpen with black cap", + "white barrel black cap markpen" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/058_markpen/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/058_markpen/base1.json new file mode 100644 index 00000000..290a6275 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/058_markpen/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "markpen", + "seen": [ + "marker pen", + "black markpen", + "plastic black marker", + "pen-sized black marker", + "black cylindrical marker", + "black pen with white label", + "white-labeled black marker pen", + "marker with tapered black ends", + "black marker with smooth barrel", + "smooth black plastic marker pen", + "black marker with white markings", + "black barrel with white label marker" + ], + "unseen": [ + "long black writing tool", + "black pen with rounded tip", + "black pen for marking surfaces" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/058_markpen/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/058_markpen/base2.json new file mode 100644 index 00000000..fe537918 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/058_markpen/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "markpen", + "seen": [ + "white pen", + "markpen with black cap", + "long white marking pen", + "pen with fine black tip", + "black capped writing marker", + "small black fine tip marker", + "marker with felt writing tip", + "plastic marker with black point", + "white marking pen with ridged cap", + "markpen featuring black rounded cap", + "white plastic marker with printed text", + "palm-sized white markpen with black features" + ], + "unseen": [ + "white body marker", + "black-tipped white marker", + "smooth cylindrical white markpen" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/058_markpen/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/058_markpen/base3.json new file mode 100644 index 00000000..bfc0b0bb --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/058_markpen/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "markpen", + "seen": [ + "black markpen", + "black and white marker pen", + "white markpen with black tip", + "markpen with pointed black tip", + "markpen with black writing end", + "markpen for writing and marking", + "black marker pen with white body", + "plastic markpen with dual colors", + "handheld marker pen with black cap", + "medium-sized cylindrical marker pen", + "marker pen with black smooth barrel", + "cylindrical markpen with smooth finish" + ], + "unseen": [ + "white pen with black ink tip", + "plastic marking pen with black cap", + "smooth white pen with black marker tip" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/058_markpen/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/058_markpen/base4.json new file mode 100644 index 00000000..8af4aaa1 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/058_markpen/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "markpen", + "seen": [ + "smooth plastic markpen", + "yellow and black markpen", + "portable cylindrical markpen", + "hexagonal yellow cap on markpen", + "markpen with yellow writing tip", + "markpen for drawing and writing", + "markpen with white printed label", + "pocket-sized cylindrical markpen", + "markpen with cone-shaped yellow tip", + "black body markpen with white labels", + "yellow-capped black and white markpen", + "black markpen with hexagonal yellow cap" + ], + "unseen": [ + "small fiber tip markpen", + "yellow and black smooth markpen", + "compact black and yellow markpen" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/058_markpen/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/058_markpen/base5.json new file mode 100644 index 00000000..33d6e0dc --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/058_markpen/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "markpen", + "seen": [ + "black markpen", + "plastic markpen", + "hand-sized markpen", + "smooth black marker", + "marker with red tip", + "red and black markpen", + "small writing markpen", + "red and black writing tool", + "markpen with red highlights", + "compact markpen for writing", + "standard cylindrical markpen", + "black casing red accents markpen" + ], + "unseen": [ + "markpen for drawing", + "markpen with rounded cap", + "red tip black-bodied markpen" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/059_pencup/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/059_pencup/base0.json new file mode 100644 index 00000000..883b7837 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/059_pencup/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "pencup", + "seen": [ + "white pencup", + "cylindrical pencup", + "pencup holding black pens", + "pencup with glossy black pens", + "simple white pencup with pens", + "white cup holding thin black pens", + "plastic pencup with smooth texture", + "white pencup holding multiple pens", + "compact pencup for organizing pens", + "glossy black pens standing in pencup", + "white pencup with writing pens inside", + "medium cylindrical container for pens" + ], + "unseen": [ + "smooth plastic pencup", + "medium-sized white pencup", + "light and smooth plastic pencup" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/059_pencup/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/059_pencup/base1.json new file mode 100644 index 00000000..21873ac0 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/059_pencup/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "pencup", + "seen": [ + "blue pencup", + "dark blue pencup", + "rectangular pencup", + "dark blue pen holder", + "medium-sized pen holder", + "dark blue plastic pencup", + "pencup with beige handles", + "rounded rectangular pencup", + "pencup with side metal bars", + "blue pencup with beige accents", + "smooth dark blue pen container", + "smooth pencup with rounded edges" + ], + "unseen": [ + "blue and beige pencup", + "blue rectangular cup for pens", + "rectangular dark blue table pencup" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/059_pencup/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/059_pencup/base2.json new file mode 100644 index 00000000..ff69c3d3 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/059_pencup/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "pencup", + "seen": [ + "white and dark gray pencup", + "white cup with dark accents", + "pen holder with inner ridges", + "smooth white cup with open top", + "medium pencup with gray bottom", + "desk holder for pens and pencils", + "container for pens and small tools", + "hand-sized cup for pencils and pens", + "cylindrical cup for desk organization", + "white plastic pen holder with grooves", + "cylindrical pencup with dark gray base", + "cylindrical pen container with ridged wall" + ], + "unseen": [ + "open top cup for office supplies", + "smooth outer cup with ridged inside", + "medium plastic cup for holding pens" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/059_pencup/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/059_pencup/base3.json new file mode 100644 index 00000000..e281c8f5 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/059_pencup/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "pencup", + "seen": [ + "cylindrical pencup", + "plastic blue pencup", + "cylindrical blue pen cup", + "medium blue pencil holder", + "blue container for pencils", + "pencup with smooth blue surface", + "desktop blue pencup with pencils", + "pencup with yellow pencils inside", + "smooth blue pencup holding pencils", + "blue cylindrical holder for pencils", + "blue plastic cup containing pencils", + "blue pen holder with included pencils" + ], + "unseen": [ + "blue pencup", + "medium-sized blue plastic pencup", + "blue pencup filled with yellow pencils" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/059_pencup/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/059_pencup/base4.json new file mode 100644 index 00000000..6cf9368f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/059_pencup/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "pencup", + "seen": [ + "desk pencup for pens", + "wooden box-shaped pencup", + "rectangular wooden pencup", + "medium brown smooth pencup", + "dark wood pencup with slots", + "pen holder with wood texture", + "wooden box pencup for stationery", + "rectangular wooden pen organizer", + "dark brown pencup with open slots", + "desktop pencup with slots for pens", + "medium-sized pencup for writing tools", + "brown pencup with multiple compartments" + ], + "unseen": [ + "brown pencup", + "rectangular pencup with grain lines", + "smooth pencup with visible wood pattern" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/059_pencup/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/059_pencup/base5.json new file mode 100644 index 00000000..38ba43ca --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/059_pencup/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "pencup", + "seen": [ + "blue pencup", + "hand-sized blue pencup", + "small light blue pencup", + "open-top pencup for pens", + "light glossy blue pencup", + "pencup with rounded edges", + "small holder for pens in blue", + "rounded rectangular blue pen holder", + "blue container with rounded corners", + "textured blue rectangular pen storage", + "rectangular pencup with rough patterns", + "plastic pencup with irregular patterns" + ], + "unseen": [ + "light blue rectangular pencup", + "smooth light blue plastic pencup", + "light blue pencup with textured sides" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/059_pencup/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/059_pencup/base6.json new file mode 100644 index 00000000..2eb77944 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/059_pencup/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "pencup", + "seen": [ + "brown pencup", + "smooth plastic pencup", + "pencup for small items", + "rectangular brown pencup", + "brown pencup with open top", + "compact rectangular pencup", + "light brown plastic pencup", + "brown pencup with beige eraser", + "pencup with smooth brown finish", + "pencup holding pens and erasers", + "pencup with black and white pens", + "rectangular plastic brown pencup" + ], + "unseen": [ + "box-shaped brown pencup", + "small plastic brown pencup", + "pencup for organizing stationery" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/060_kitchenpot/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/060_kitchenpot/base0.json new file mode 100644 index 00000000..16127de6 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/060_kitchenpot/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "kitchenpot", + "seen": [ + "gray kitchenpot", + "metal kitchenpot", + "cylindrical kitchenpot", + "smooth gray kitchenpot", + "medium-sized kitchenpot", + "dark gray stove kitchenpot", + "round kitchenpot with handles", + "medium kitchenpot made of metal", + "kitchenpot with smooth exterior", + "gray cylindrical metal kitchenpot", + "cooking kitchenpot with side grips", + "kitchenpot for boiling and cooking" + ], + "unseen": [ + "dark gray cooking kitchenpot", + "kitchenpot with two side handles", + "gray kitchenpot with rounded edges" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/060_kitchenpot/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/060_kitchenpot/base1.json new file mode 100644 index 00000000..94017ebd --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/060_kitchenpot/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "kitchenpot", + "seen": [ + "gray kitchenpot", + "blue-lid kitchenpot", + "gray cylindrical pot", + "metal pot with dark handles", + "cooking pot with two handles", + "pots for boiling and cooking", + "matte gray pot with durable lid", + "gray pot with blue circular lid", + "blue kitchenpot lid with black knob", + "metal kitchenpot with brown handles", + "kitchenpot with plastic knob on lid", + "round blue-lid kitchenpot for stovetop" + ], + "unseen": [ + "medium stove pot", + "round pot with blue top", + "kitchenpot with smooth matte body" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/060_kitchenpot/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/060_kitchenpot/base2.json new file mode 100644 index 00000000..bfaccb18 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/060_kitchenpot/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "kitchenpot", + "seen": [ + "silver kitchenpot", + "silver pot with black grips", + "silver pot with matching lid", + "medium silver pot for cooking", + "metallic pot with matte finish", + "rounded silver pot with handles", + "rounded pot with curved handles", + "metallic pot with plastic handles", + "cooking pot with snug-fitting lid", + "silver kitchenpot with black grips", + "kitchenpot with smooth metallic lid", + "medium-sized pot with plastic grips" + ], + "unseen": [ + "kitchenpot for stovetop boiling", + "medium kitchenpot with black handles", + "rounded cooking pot with two handles" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/060_kitchenpot/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/060_kitchenpot/base3.json new file mode 100644 index 00000000..66002720 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/060_kitchenpot/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "kitchenpot", + "seen": [ + "teal pot for boiling and cooking", + "blue cylinder pot with smooth lid", + "kitchenpot with flat lid and knob", + "cooking pot with two blue handles", + "medium-sized teal blue cooking pot", + "round kitchenpot with two side grips", + "medium teal-colored pot with flat lid", + "smooth teal kitchenpot for stovetop use", + "kitchenpot with two sturdy blue handles", + "blue kitchenpot with smooth matte sides", + "medium teal blue kitchenpot with handles", + "round teal cooking pot with matte texture" + ], + "unseen": [ + "blue kitchenpot", + "teal blue cooking pot", + "blue pot with cylindrical body and lid" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/060_kitchenpot/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/060_kitchenpot/base4.json new file mode 100644 index 00000000..f950c2ea --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/060_kitchenpot/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "kitchenpot", + "seen": [ + "black kitchenpot", + "black metal kitchenpot", + "kitchenpot for boiling food", + "black kitchenpot with flat lid", + "black kitchenpot with round lid", + "kitchenpot with metallic round lid", + "black cooking pot with silver handles", + "medium kitchenpot with smooth surface", + "cooking kitchenpot with sturdy handles", + "black kitchenpot with silver handle bars", + "black kitchenpot with plastic lid handle", + "black kitchenpot with silver and black handles" + ], + "unseen": [ + "round black kitchenpot", + "shiny black kitchenpot", + "medium black kitchenpot with lid" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/060_kitchenpot/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/060_kitchenpot/base5.json new file mode 100644 index 00000000..aa04bb0c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/060_kitchenpot/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "kitchenpot", + "seen": [ + "gray kitchenpot", + "metal cooking pot", + "pot with round lid", + "gray pot for boiling food", + "rounded pot with metal lid", + "medium gray pot with handles", + "round pot with dark gray knob", + "kitchenpot with smooth texture", + "dark gray pot with side handles", + "pot with lid and sturdy handles", + "kitchenpot used for cooking meals", + "medium-sized smooth gray cooking pot" + ], + "unseen": [ + "metal pot with curved handles", + "smooth gray pot with matte finish", + "cylindrical kitchenpot for cooking" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/060_kitchenpot/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/060_kitchenpot/base6.json new file mode 100644 index 00000000..955dd23c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/060_kitchenpot/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "kitchenpot", + "seen": [ + "grey kitchenpot", + "round kitchenpot", + "matte-finished kitchenpot", + "kitchenpot with black base", + "kitchenpot with two side handles", + "light grey kitchenpot with dark top", + "smooth kitchenpot with black button", + "kitchenpot with sturdy plastic body", + "kitchenpot with red indicator light", + "cylindrical kitchenpot with flat edges", + "grey kitchenpot with dark circle inside", + "kitchenpot with black rectangular panel" + ], + "unseen": [ + "medium kitchenpot", + "kitchenpot for cooking food", + "compact kitchenpot with lid-like surface" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/060_kitchenpot/base7.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/060_kitchenpot/base7.json new file mode 100644 index 00000000..32dd8889 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/060_kitchenpot/base7.json @@ -0,0 +1,22 @@ +{ + "raw_description": "kitchenpot", + "seen": [ + "silver kitchenpot", + "medium-sized kitchenpot", + "shiny silver kitchenpot", + "pot for stovetop cooking", + "metal kitchenpot with lid", + "stainless steel kitchenpot", + "kitchenpot with silver lid", + "kitchenpot with side handles", + "kitchenpot with smooth finish", + "kitchenpot with round metal lid", + "kitchenpot with two curved handles", + "cylindrical kitchenpot with handles" + ], + "unseen": [ + "round kitchenpot", + "cooking kitchenpot", + "smooth steel kitchenpot" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/061_battery/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/061_battery/base0.json new file mode 100644 index 00000000..a68c182e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/061_battery/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "battery", + "seen": [ + "yellow tip battery", + "black and yellow battery", + "handheld cylindrical battery", + "battery with black body yellow top", + "battery with two-tone color scheme", + "small round battery smooth texture", + "battery with white markings on side", + "battery labeled with yellow details", + "round yellow and black energy battery", + "cylindrical battery with smooth surface", + "compact battery with black main section", + "battery cylinder with white printed label" + ], + "unseen": [ + "metal battery smooth and round", + "small battery with metal coating", + "power source battery cylinder shape" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/061_battery/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/061_battery/base1.json new file mode 100644 index 00000000..cccf4a5a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/061_battery/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "battery", + "seen": [ + "blue battery", + "small blue AA battery", + "blue cylindrical battery", + "blue battery for devices", + "battery with rounded ends", + "battery with white markings", + "AA battery with metal casing", + "cylinder-shaped power battery", + "cylindrical blue power battery", + "blue battery with white letters", + "blue battery with smooth surface", + "metal battery with smooth texture" + ], + "unseen": [ + "AA battery", + "small handheld power battery", + "blue battery with white text around" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/061_battery/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/061_battery/base2.json new file mode 100644 index 00000000..64f204cd --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/061_battery/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "battery", + "seen": [ + "battery labeled Delipow", + "yellow and green battery", + "palm-sized green battery", + "flat-ended cylindrical battery", + "metal battery with printed text", + "green battery with white writing", + "yellow-topped cylindrical battery", + "battery with glossy green surface", + "small power battery with flat base", + "small green battery with yellow ends", + "smooth metal green and yellow battery", + "fully cylindrical battery with yellow trims" + ], + "unseen": [ + "cylindrical battery", + "green battery with rounded edges", + "metal battery with printed markings" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/061_battery/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/061_battery/base3.json new file mode 100644 index 00000000..7d7882a7 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/061_battery/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "battery", + "seen": [ + "black cylindrical battery", + "hand-sized black power battery", + "black battery with yellow bands", + "battery with top and flat bottom", + "black battery with white end cap", + "battery with yellow and red labels", + "black battery tube with red markings", + "printed battery with bold yellow text", + "battery with smooth texture and labels", + "battery cylinder with shiny black plastic", + "black handheld battery with smooth surface", + "smooth cylindrical battery with printed details" + ], + "unseen": [ + "battery cylinder with metallic body", + "standard-sized cylindrical black battery", + "cylindrical power battery with red accents" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/061_battery/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/061_battery/base4.json new file mode 100644 index 00000000..41818176 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/061_battery/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "battery", + "seen": [ + "black battery", + "rectangular battery", + "compact battery block", + "plastic and metal battery", + "medium-sized black battery", + "battery with red connectors", + "black battery with red terminals", + "black battery with visible bolts", + "battery with two visible connectors", + "black power battery with cable ports", + "smooth battery with a rectangular form", + "black rectangular battery with grooves" + ], + "unseen": [ + "power-providing black battery", + "battery with a grooved surface", + "battery with top barcode label" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/061_battery/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/061_battery/base5.json new file mode 100644 index 00000000..f4a90a91 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/061_battery/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "battery", + "seen": [ + "yellow battery", + "smooth cylinder battery", + "battery with white label", + "cylindrical yellow battery", + "battery for powering devices", + "yellow battery with flat ends", + "small battery with metal casing", + "battery with black circular end", + "yellow battery with labeled text", + "round battery with flat terminals", + "battery with printed white wrapping", + "small yellow battery labeled on side" + ], + "unseen": [ + "metallic yellow battery", + "yellow battery with white stripe", + "palm-sized yellow and white battery" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base0.json new file mode 100644 index 00000000..af61175b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "plasticbox", + "seen": [ + "gray plasticbox", + "gray box made of plastic", + "smooth light gray plasticbox", + "plasticbox with smooth surface", + "gray plasticbox hand-held size", + "smooth gray box made of plastic", + "light gray rectangular plasticbox", + "light gray plasticbox for storage", + "rectangular gray storage plasticbox", + "plastic storage box light gray color", + "rectangular plasticbox with clear edges", + "plasticbox rectangular with smooth finish" + ], + "unseen": [ + "medium-sized gray plasticbox", + "box with smooth plastic finish", + "plasticbox with rectangular shape" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base1.json new file mode 100644 index 00000000..188d285d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "plasticbox", + "seen": [ + "red plasticbox", + "glossy red plasticbox", + "bright red medium plasticbox", + "medium-sized storage plasticbox", + "red plasticbox with shiny finish", + "solid red plastic rectangular box", + "plasticbox with flat top and sides", + "red plasticbox with angular corners", + "smooth plasticbox with angular shape", + "medium rectangular storage plasticbox", + "red plasticbox with patterned exterior", + "plasticbox shaped like a rectangular prism" + ], + "unseen": [ + "sleek smooth red plasticbox", + "plasticbox with curved edges", + "plain rectangular plasticbox" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base10.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base10.json new file mode 100644 index 00000000..e9e6e6c5 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base10.json @@ -0,0 +1,22 @@ +{ + "raw_description": "plasticbox", + "seen": [ + "blue plasticbox", + "blue box for storage", + "plasticbox with handles", + "medium box with blue color", + "blue box for organizing items", + "medium blue box with perforations", + "medium blue rectangular plasticbox", + "plasticbox with smooth blue surface", + "blue plasticbox with holding handles", + "rectangular plasticbox with side holes", + "blue plasticbox with rectangular shape", + "medium rectangular perforated plasticbox" + ], + "unseen": [ + "perforated plasticbox", + "smooth blue plasticbox", + "carrying plasticbox made of blue plastic" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base2.json new file mode 100644 index 00000000..e8388711 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "plasticbox", + "seen": [ + "smooth plasticbox", + "light gray plasticbox", + "rectangular plasticbox", + "simple gray plasticbox for use", + "gray plasticbox for storing items", + "plasticbox for organizing small items", + "rectangular plasticbox with open face", + "plasticbox with smooth light gray finish", + "light gray rectangular storage plasticbox", + "plasticbox with smooth rectangular structure", + "medium rectangular plasticbox with open front", + "medium light gray rectangular storage plasticbox" + ], + "unseen": [ + "medium-sized plasticbox", + "plasticbox with open front side", + "smooth gray plasticbox for storage" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base3.json new file mode 100644 index 00000000..233b1336 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "plasticbox", + "seen": [ + "light blue plasticbox", + "plasticbox with hinged lid", + "boxy light blue plasticbox", + "plasticbox for carrying items", + "plasticbox for keeping items safe", + "smooth blue plasticbox with handle", + "blue plasticbox for storing things", + "durable plasticbox with side handle", + "plasticbox with smooth plastic texture", + "plasticbox with rounded corners and lid", + "rectangular plasticbox with rounded edges", + "light blue rectangular storage plasticbox" + ], + "unseen": [ + "medium-sized smooth plasticbox", + "portable light blue storage plasticbox", + "rectangular plasticbox with locking clips" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base4.json new file mode 100644 index 00000000..5f064e4d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "plasticbox", + "seen": [ + "green box", + "plasticbox", + "solid green container", + "green plastic storage box", + "green box with grid at base", + "plasticbox with smooth sides", + "medium rectangular plasticbox", + "plasticbox with ridged bottom", + "medium green plastic container", + "plasticbox for organizing items", + "solid green rectangular plasticbox", + "smooth green plasticbox with grid base" + ], + "unseen": [ + "rectangular green plasticbox", + "medium green box for storage", + "rectangular storage plasticbox" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base5.json new file mode 100644 index 00000000..0a677d7a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "plasticbox", + "seen": [ + "blue plasticbox", + "medium blue plasticbox", + "medium storage plasticbox", + "deep blue box for storage", + "blue box with angled edges", + "smooth blue hard plasticbox", + "blue box for storing things", + "medium box with angled sides", + "plasticbox with grooved sides", + "deep blue plasticbox with grooves", + "rectangular storage box with grooves", + "blue plastic rectangular storage box" + ], + "unseen": [ + "rectangular plasticbox", + "hard plastic rectangular box", + "smooth exterior blue plasticbox" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base6.json new file mode 100644 index 00000000..d47085a2 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "plasticbox", + "seen": [ + "blue plasticbox", + "plasticbox with open top", + "smooth medium plasticbox", + "open-top blue plasticbox", + "plasticbox made of hard plastic", + "plasticbox for organizing items", + "plasticbox with ridges on bottom", + "blue plasticbox shaped like a box", + "blue rectangular prism plasticbox", + "blue plasticbox with raised edges", + "blue rectangular smooth plasticbox", + "medium plasticbox with smooth surface" + ], + "unseen": [ + "medium rectangular blue plasticbox", + "rectangular blue storage plasticbox", + "rectangular blue plasticbox for storage" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base7.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base7.json new file mode 100644 index 00000000..8cc69824 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base7.json @@ -0,0 +1,22 @@ +{ + "raw_description": "plasticbox", + "seen": [ + "gray plasticbox", + "smooth gray plasticbox", + "light gray storage plasticbox", + "plasticbox with hollow handles", + "rectangular box made of plastic", + "medium plasticbox for holding items", + "plasticbox with two oval side handles", + "plasticbox with smooth outside texture", + "medium-sized gray plasticbox for storage", + "rectangular gray box with cut-out handles", + "smooth rectangular plasticbox for storing things", + "light gray plasticbox with ribbed pattern inside" + ], + "unseen": [ + "rectangular plasticbox", + "plasticbox with open ends", + "light-colored plasticbox with ridged interior" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base8.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base8.json new file mode 100644 index 00000000..5a4d55e9 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base8.json @@ -0,0 +1,22 @@ +{ + "raw_description": "plasticbox", + "seen": [ + "medium plasticbox", + "solid blue plasticbox", + "rectangular plasticbox", + "plasticbox for storage", + "box-shaped blue plasticbox", + "plasticbox with rigid edges", + "blue box with black corners", + "shelf-sized blue plasticbox", + "plasticbox with flat smooth sides", + "plasticbox with black corner edges", + "plasticbox with rectangular handles", + "blue rectangular storage plasticbox" + ], + "unseen": [ + "blue plasticbox", + "smooth blue plasticbox", + "plasticbox with smooth sides" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base9.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base9.json new file mode 100644 index 00000000..9d70d882 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/062_plasticbox/base9.json @@ -0,0 +1,22 @@ +{ + "raw_description": "plasticbox", + "seen": [ + "blue plasticbox", + "rectangular plasticbox", + "plasticbox for storage", + "smooth blue box plasticbox", + "medium-sized blue plasticbox", + "medium blue storage plasticbox", + "plasticbox with smooth texture", + "plasticbox made of blue plastic", + "rectangular bright blue plasticbox", + "plasticbox with smooth blue surface", + "bright colored plasticbox for storage", + "blue plasticbox with rectangular shape" + ], + "unseen": [ + "bright blue rectangular plasticbox", + "medium rectangular cuboid plasticbox", + "bright blue storage plasticbox container" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base0.json new file mode 100644 index 00000000..6aa413d9 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tabletrashbin", + "seen": [ + "orange tabletrashbin", + "trashbin for small spaces", + "small orange tabletrashbin", + "tabletrashbin curved edges", + "square-shaped tabletrashbin", + "smooth square tabletrashbin", + "tabletrashbin with open top", + "square orange tabletrashbin", + "bright orange tabletrashbin", + "compact orange tabletrashbin", + "orange plastic tabletrashbin", + "plastic tabletrashbin gray edge" + ], + "unseen": [ + "tabletrashbin with gray rim", + "tabletrashbin for table trash", + "tabletrashbin smooth plastic body" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base1.json new file mode 100644 index 00000000..7fc647ca --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tabletrashbin", + "seen": [ + "small tabletop trashbin", + "gray compact tabletrashbin", + "rounded trashbin for tables", + "smooth plastic tabletrashbin", + "small light gray tabletrashbin", + "tabletrashbin with side latches", + "rounded edge gray tabletrashbin", + "light gray rectangular trashbin", + "tabletrashbin with gray latches", + "tabletrashbin with secure side clips", + "tabletrashbin made of smooth plastic", + "rectangular tabletrashbin with rounded corners" + ], + "unseen": [ + "compact plastic tabletrashbin", + "light gray trashbin for tables", + "small rectangular tabletrashbin" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base10.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base10.json new file mode 100644 index 00000000..22222392 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base10.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tabletrashbin", + "seen": [ + "gray plastic trashbin", + "open-top tabletrashbin", + "light gray tabletrashbin", + "small light gray waste bin", + "bucket-shaped gray trashbin", + "small rectangular tabletrashbin", + "tabletrashbin for holding waste", + "small rectangular bin for waste", + "light gray trashbin with open top", + "tabletrashbin with smooth grooves", + "light gray bucket-shaped waste bin", + "rectangular tabletop waste container" + ], + "unseen": [ + "compact tabletop trashbin", + "smooth plastic tabletrashbin", + "desktop tabletrashbin for small trash" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base2.json new file mode 100644 index 00000000..937c4f21 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tabletrashbin", + "seen": [ + "table trash bin", + "smooth gray trash bin", + "rounded lid trash bin", + "gray bin with round lid", + "plastic bin for desk trash", + "smooth light gray trash bin", + "cylindrical bin with handles", + "gray bin with embossed sides", + "small trash bin for tabletop", + "desktop trash bin light gray", + "small bin with lid and ridges", + "light gray bin for small waste" + ], + "unseen": [ + "light gray trash bin", + "small cylinder trash container", + "trash bin with small round knob" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base3.json new file mode 100644 index 00000000..bfe9eb49 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tabletrashbin", + "seen": [ + "gray tabletrashbin", + "small tabletrashbin", + "smooth plastic tabletrashbin", + "tabletrashbin with black specks", + "tabletrashbin for tabletop trash", + "tabletrashbin with tapered sides", + "black-specked gray tabletrashbin", + "dark gray cylindrical tabletrashbin", + "compact speckled surface tabletrashbin", + "tabletrashbin with slightly flared top", + "plastic tabletrashbin with dark pattern", + "small cylindrical speckled tabletrashbin" + ], + "unseen": [ + "portable gray speckled tabletrashbin", + "tabletrashbin for holding small trash", + "little tabletrashbin with smooth texture" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base4.json new file mode 100644 index 00000000..c965f938 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tabletrashbin", + "seen": [ + "dark blue trash bin", + "rounded blue bin for trash", + "palm-sized trash container", + "matte plastic dark blue bin", + "tabletrashbin with swivel lid", + "tabletrashbin made of plastic", + "tabletrashbin for small trash", + "compact dark blue cylinder bin", + "tiny trash bin with swivel lid", + "rounded cylindrical tabletrashbin", + "rounded tabletrashbin with curved top", + "plastic tabletrashbin with smooth surface" + ], + "unseen": [ + "small blue trash bin", + "blue miniature trash bin", + "blue bin with flip-top lid" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base5.json new file mode 100644 index 00000000..6e7f5bea --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tabletrashbin", + "seen": [ + "trash bin for tables", + "small purple trash bin", + "hand-sized tabletrashbin", + "tiny trash bin in purple", + "smooth purple tabletrashbin", + "compact plastic tabletrashbin", + "plastic bin with ribbed sides", + "tabletrashbin with rounded edges", + "dark-colored plastic tabletrashbin", + "purple tabletrashbin with small lid", + "small smooth tabletrashbin in purple", + "rounded purple tabletrashbin for table use" + ], + "unseen": [ + "purple tabletrashbin", + "dark purple tabletrashbin with lid", + "tabletrashbin with textured ribbed surface" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base6.json new file mode 100644 index 00000000..ba74a983 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tabletrashbin", + "seen": [ + "small tabletrashbin", + "rounded tabletrashbin", + "light brown tabletrashbin", + "wooden-look tabletrashbin", + "tabletrashbin for tabletop use", + "tabletrashbin with hollow inside", + "tabletrashbin with rounded edges", + "tabletrashbin with smooth surface", + "tabletrashbin with visible wood grain", + "tabletrashbin made from thin material", + "compact tabletrashbin for small spaces", + "tabletrashbin designed for holding trash" + ], + "unseen": [ + "tiny tabletrashbin", + "light brown hollow tabletrashbin", + "tabletrashbin shaped like a bowl" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base7.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base7.json new file mode 100644 index 00000000..d84edac2 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base7.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tabletrashbin", + "seen": [ + "plastic tabletrashbin", + "open-top tabletrashbin", + "small tabletop trashbin", + "open gray tabletrashbin", + "light gray tabletrashbin", + "round gray tabletrashbin", + "simple tabletop trashbin", + "cylindrical tabletrashbin", + "light gray waste container", + "minimalist gray tabletrashbin", + "light gray smooth tabletrashbin", + "light gray cylindrical trashbin" + ], + "unseen": [ + "small smooth gray trashbin", + "gray trashbin for tabletop", + "tabletrashbin made of plastic" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base8.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base8.json new file mode 100644 index 00000000..526aa6c2 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base8.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tabletrashbin", + "seen": [ + "light gray trash bin", + "small trash container", + "light gray tabletrashbin", + "lightweight trash container", + "smooth plastic tabletrashbin", + "tabletrashbin made of plastic", + "compact bin with rounded edges", + "tabletrashbin with yellow symbol", + "small container with yellow logo", + "compact rectangular tabletrashbin", + "tabletrashbin with grooves on sides", + "smooth tabletrashbin for small waste" + ], + "unseen": [ + "rectangular plastic bin", + "tabletrashbin for tabletop use", + "rectangular light gray trash bin" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base9.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base9.json new file mode 100644 index 00000000..88226b65 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/063_tabletrashbin/base9.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tabletrashbin", + "seen": [ + "tabletrashbin", + "small white trashbin", + "compact tabletop trashbin", + "white round tabletrashbin", + "white cylindrical waste bin", + "smooth plastic tabletrashbin", + "mini white trashbin for desk", + "simple white plastic trashbin", + "white bin with flared top edge", + "desk-sized white tabletrashbin", + "cylindrical small white trashbin", + "cylindrical tabletrashbin with flared rim" + ], + "unseen": [ + "small bin for table waste", + "plastic tiny trashbin for tables", + "lightweight round tabletop trashbin" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/064_msg/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/064_msg/base0.json new file mode 100644 index 00000000..e9f5a80e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/064_msg/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "msg", + "seen": [ + "msg bottle", + "yellow labeled msg bottle", + "msg bottle cylindrical shape", + "msg plastic bottle small size", + "msg drink bottle rounded edges", + "small brown bottle with red cap", + "msg bottle cap bright red color", + "brown body msg with yellow print", + "msg container with bright red cap", + "smooth plastic msg beverage bottle", + "msg labeled drink container body brown", + "round-topped msg bottle with yellow label" + ], + "unseen": [ + "smooth brown msg bottle", + "hand-sized msg drink bottle", + "red cap on brown msg container" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/064_msg/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/064_msg/base1.json new file mode 100644 index 00000000..ff94306f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/064_msg/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "can", + "seen": [ + "metal can", + "yellow can", + "smooth yellow can", + "yellow cylindrical can", + "hand-sized yellow food can", + "yellow can with white label", + "metallic can with sealed top", + "metal can with sealed bottom", + "yellow can with glossy finish", + "sealed can with chicken soup label", + "cylindrical metal can with top lid", + "yellow can with white and black text" + ], + "unseen": [ + "food can", + "metal can with round edges", + "cylindrical chicken soup can" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/064_msg/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/064_msg/base2.json new file mode 100644 index 00000000..25428cab --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/064_msg/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "msg", + "seen": [ + "yellow msg tube", + "bright yellow msg can", + "tube with msg branding", + "smooth plastic msg tube", + "yellow tube for msg powder", + "medium tube with msg label", + "msg tube with printed label", + "msg bottle with rounded cap", + "msg cylinder with orange top", + "yellow plastic msg dispenser", + "glossy msg container with lid", + "msg powder tube with orange cap" + ], + "unseen": [ + "msg with orange lid", + "handheld yellow msg canister", + "cylindrical yellow msg container" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/064_msg/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/064_msg/base3.json new file mode 100644 index 00000000..30a0ae07 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/064_msg/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "msg", + "seen": [ + "yellow msg cylinder", + "smooth yellow msg can", + "msg can with rounded edges", + "yellow metal msg can with text", + "handheld msg can with green lid", + "medium-size msg storage canister", + "msg container with flat green lid", + "yellow msg canister with sealed top", + "bright msg can with colorful images", + "cylindrical green-and-yellow msg can", + "msg cylinder with green base and lid", + "green-lidded msg cylindrical container" + ], + "unseen": [ + "cylinder with msg branding", + "msg can with printed designs", + "yellow metal cylinder with msg label" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/064_msg/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/064_msg/base4.json new file mode 100644 index 00000000..c04a7fc0 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/064_msg/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "msg", + "seen": [ + "yellow cylinder", + "msg with red cap", + "plastic container", + "cylinder with red lid", + "round yellow container", + "cylindrical yellow jar", + "msg in plastic cylinder", + "medium yellow container", + "msg logo on yellow body", + "yellow plastic msg bottle", + "msg jar with shiny surface", + "yellow cylinder with smooth surface" + ], + "unseen": [ + "msg cylinder", + "msg powder container", + "yellow cylinder with writing" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/064_msg/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/064_msg/base5.json new file mode 100644 index 00000000..9db0457d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/064_msg/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "msg", + "seen": [ + "msg with green lid", + "yellow cylindrical msg", + "bright yellow msg cylinder", + "msg with green top and base", + "medium-sized yellow msg container", + "yellow msg can with smooth texture", + "msg cylinder for culinary purposes", + "msg tub with yellow and green colors", + "yellow msg canister with printed text", + "msg food container with chicken image", + "msg packaging with bright yellow body", + "green and yellow cylindrical msg holder" + ], + "unseen": [ + "msg can with chicken label", + "msg container for kitchen use", + "msg can medium size smooth finish" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/065_soy-sauce/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/065_soy-sauce/base0.json new file mode 100644 index 00000000..0953e23b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/065_soy-sauce/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "soy-sauce", + "seen": [ + "soy-sauce bottle", + "hand-sized soy-sauce bottle", + "pink capped soy-sauce bottle", + "rounded black soy-sauce bottle", + "cylindrical soy-sauce container", + "soy-sauce bottle with pink screw cap", + "medium soy-sauce bottle with pink cap", + "yellow label bottle holding soy-sauce", + "yellow labeled soy-sauce plastic bottle", + "black bottle with yellow soy-sauce label", + "plastic soy-sauce bottle with paper label", + "plastic soy-sauce bottle with orange gradient label" + ], + "unseen": [ + "smooth black soy-sauce bottle", + "yellow and black soy-sauce bottle", + "medium soy-sauce container with ribbed cap" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/065_soy-sauce/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/065_soy-sauce/base1.json new file mode 100644 index 00000000..794f563e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/065_soy-sauce/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "soy-sauce", + "seen": [ + "soy-sauce bottle", + "black bottle with red cap", + "red-lidded soy-sauce bottle", + "glossy black soy-sauce bottle", + "medium black soy-sauce container", + "smooth glass soy-sauce packaging", + "yellow-labelled soy-sauce bottle", + "smooth glass bottle for soy sauce", + "medium-sized glass soy-sauce bottle", + "soy-sauce bottle with rectangular label", + "handheld soy-sauce bottle with ridged cap", + "cylindrical soy-sauce bottle with long neck" + ], + "unseen": [ + "glass soy-sauce bottle with yellow label", + "soy-sauce bottle with bright yellow sticker", + "soy-sauce bottle sealed with red plastic cap" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/065_soy-sauce/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/065_soy-sauce/base2.json new file mode 100644 index 00000000..33633a94 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/065_soy-sauce/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "soy-sauce", + "seen": [ + "soy-sauce bottle", + "small soy-sauce container", + "palm-sized soy-sauce bottle", + "red-lidded soy-sauce bottle", + "curved bottle for soy-sauce", + "black bottle with slim neck", + "small black bottle with red lid", + "cylindrical bottle for soy-sauce", + "bottle of soy-sauce with red top", + "black plastic bottle for soy-sauce", + "soy-sauce bottle with smooth surface", + "soy-sauce bottle with glossy black body" + ], + "unseen": [ + "black bottle with red cap", + "soy-sauce bottle with red labels", + "compact soy-sauce bottle with labels" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/065_soy-sauce/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/065_soy-sauce/base3.json new file mode 100644 index 00000000..8f9bc725 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/065_soy-sauce/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "soy-sauce", + "seen": [ + "dark soy-sauce bottle", + "bottle with red label", + "handheld soy-sauce bottle", + "rounded soy-sauce container", + "medium-sized soy-sauce bottle", + "dark brown liquid inside bottle", + "medium bottle with textured cap", + "round-shaped soy-sauce container", + "smooth plastic soy-sauce container", + "soy-sauce bottle with pull-tab cap", + "plastic soy-sauce bottle with spout", + "red and white lidded soy-sauce bottle" + ], + "unseen": [ + "red-labeled soy-sauce bottle", + "bottle of soy sauce for pouring", + "dark bottle with red plastic lid" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/065_soy-sauce/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/065_soy-sauce/base4.json new file mode 100644 index 00000000..249f026c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/065_soy-sauce/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "soy-sauce", + "seen": [ + "black body soy-sauce bottle", + "palm-sized soy-sauce bottle", + "easy-to-grip soy-sauce bottle", + "dark-colored soy-sauce bottle", + "bright red cap soy-sauce bottle", + "smooth plastic soy-sauce bottle", + "bottle with red cap and black body", + "shiny teardrop soy-sauce container", + "soy-sauce bottle with golden label", + "soy-sauce bottle with smooth surface", + "small teardrop-shaped soy-sauce bottle", + "handheld glossy black soy-sauce container" + ], + "unseen": [ + "black soy-sauce bottle", + "compact soy-sauce bottle", + "red capped soy-sauce bottle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/066_vinegar/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/066_vinegar/base0.json new file mode 100644 index 00000000..be83efcb --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/066_vinegar/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "vinegar", + "seen": [ + "vinegar bottle", + "dark vinegar container", + "dark cylinder with red cap", + "red-labeled vinegar bottle", + "smooth black bottle with red lid", + "vinegar bottle for pouring liquid", + "vinegar bottle with smooth texture", + "plastic vinegar bottle with red lid", + "black vinegar container with white top", + "plastic vinegar bottle with curved grip", + "medium-sized dark plastic vinegar bottle", + "dark bottle with textured handle for vinegar" + ], + "unseen": [ + "plastic body with red label", + "curved handle vinegar bottle", + "medium black vinegar bottle with handle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/066_vinegar/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/066_vinegar/base1.json new file mode 100644 index 00000000..9c8f4101 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/066_vinegar/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "vinegar", + "seen": [ + "black bottle with red label", + "smooth glass vinegar bottle", + "yellow capped vinegar bottle", + "textured cap on vinegar bottle", + "black vinegar container with cap", + "medium glass bottle with vinegar", + "glass bottle with red-yellow label", + "black glass bottle with golden cap", + "cylindrical dark bottle for vinegar", + "vinegar bottle with patterned label", + "vinegar bottle with smooth black finish", + "handheld vinegar bottle with narrow neck" + ], + "unseen": [ + "vinegar bottle", + "red and yellow labeled vinegar bottle", + "medium-sized cylindrical vinegar bottle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/066_vinegar/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/066_vinegar/base2.json new file mode 100644 index 00000000..50cc1739 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/066_vinegar/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "vinegar", + "seen": [ + "plastic bottle for vinegar", + "smooth black vinegar bottle", + "medium-sized vinegar container", + "glossy black vinegar container", + "vinegar bottle with curved neck", + "hand-sized black vinegar bottle", + "black bottle for vinegar liquid", + "vinegar bottle with yellow writing", + "vinegar in a bottle with red label", + "bottle with narrow neck for vinegar", + "red and yellow labeled vinegar bottle", + "vinegar bottle with tall cylindrical shape" + ], + "unseen": [ + "black vinegar bottle", + "vinegar bottle with yellow label", + "black bottle with red and yellow label" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/067_steamer/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/067_steamer/base0.json new file mode 100644 index 00000000..56bc0600 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/067_steamer/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "steamer", + "seen": [ + "food steamer", + "plastic steamer", + "circular steamer", + "smooth yellow steamer", + "steamer with small holes", + "beige and yellow steamer", + "yellow-rimmed food steamer", + "yellow steamer for cooking", + "steamer with grid-like base", + "medium-sized circular steamer", + "light beige steamer with holes", + "perforated plastic steamer for steaming" + ], + "unseen": [ + "yellow steamer", + "steamer with perforated base", + "round steamer with yellow rim" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/067_steamer/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/067_steamer/base1.json new file mode 100644 index 00000000..b77fac45 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/067_steamer/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "steamer", + "seen": [ + "bamboo steamer", + "round bamboo steamer", + "two-layer bamboo steamer", + "steamer with slatted base", + "hand-sized bamboo steamer", + "yellow cylindrical steamer", + "cylindrical steamer with lid", + "round steamer with smooth sides", + "medium steamer for cooking food", + "natural wood grain bamboo steamer", + "steamer with smooth bamboo texture", + "light yellow cylindrical cooking steamer" + ], + "unseen": [ + "light yellow steamer", + "steamer with removable lid", + "round steamer with slatted interior" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/067_steamer/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/067_steamer/base2.json new file mode 100644 index 00000000..0d0c9773 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/067_steamer/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "steamer", + "seen": [ + "round steamer", + "smooth steamer", + "plastic steamer", + "light brown steamer", + "steamer with white rim", + "medium shallow steamer", + "brown steamer for steaming", + "round plastic food steamer", + "light brown steamer with holes", + "round steamer with perforated top", + "steamer with diamond-shaped holes", + "light brown steamer with white edge" + ], + "unseen": [ + "food steamer", + "white-rimmed steamer", + "steamer for cooking food" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/068_boxdrink/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/068_boxdrink/base0.json new file mode 100644 index 00000000..7a53b1fb --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/068_boxdrink/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "boxdrink", + "seen": [ + "white boxdrink", + "brown and white boxdrink", + "boxdrink with brown label", + "cylindrical white boxdrink", + "medium-sized white boxdrink", + "logo-printed white boxdrink", + "white boxdrink with sturdy lid", + "handheld boxdrink with flat top", + "white boxdrink with printed logo", + "drink container with tapered shape", + "drink box with brown circular print", + "smooth boxdrink for holding liquids" + ], + "unseen": [ + "white boxdrink with flat lid", + "paper boxdrink with smooth surface", + "white drink box with cone-like body" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/068_boxdrink/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/068_boxdrink/base1.json new file mode 100644 index 00000000..3b6acb53 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/068_boxdrink/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "boxdrink", + "seen": [ + "orange boxdrink", + "boxdrink with green cap", + "orange cylindrical boxdrink", + "green cap atop orange boxdrink", + "orange boxdrink with fruit print", + "boxdrink for storing liquid juice", + "smooth boxdrink with printed label", + "boxdrink shaped like a cone bottle", + "medium-sized orange plastic boxdrink", + "plastic boxdrink with smooth texture", + "orange bottle tapering toward green cap", + "handheld orange boxdrink with sealed top" + ], + "unseen": [ + "boxdrink for juice storage", + "bright orange boxdrink with white label", + "bottle-shaped boxdrink labeled with fruits" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/068_boxdrink/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/068_boxdrink/base2.json new file mode 100644 index 00000000..719f7262 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/068_boxdrink/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "boxdrink", + "seen": [ + "yellow bottle", + "boxdrink bottle", + "smooth bottle with red cap", + "bottle with red screw-on cap", + "rounded yellow drink container", + "plastic bottle with juice inside", + "boxdrink with smooth plastic body", + "yellow plastic bottle with red cap", + "medium yellow bottle with drink label", + "medium yellow boxdrink plastic bottle", + "yellow bottle with orange drink label", + "yellow drink bottle with colorful design" + ], + "unseen": [ + "yellow bottle for holding boxdrink", + "yellow bottle with printed wrapper", + "curved plastic bottle for drink storage" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/068_boxdrink/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/068_boxdrink/base3.json new file mode 100644 index 00000000..8bf8fabf --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/068_boxdrink/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "boxdrink", + "seen": [ + "yellow boxdrink", + "medium boxdrink bottle", + "boxdrink with green cap", + "boxdrink with floral label", + "hand-sized boxdrink bottle", + "boxdrink with barcode on back", + "smooth yellow boxdrink bottle", + "boxdrink bottle with firm grip", + "light yellow boxdrink with label", + "boxdrink with white screw-on lid", + "smooth plastic boxdrink container", + "light yellow rectangular boxdrink" + ], + "unseen": [ + "yellow boxdrink with illustrations", + "yellow boxdrink with green and white cap", + "rectangular boxdrink with rounded corners" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/069_vagetable/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/069_vagetable/base0.json new file mode 100644 index 00000000..6f014eb1 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/069_vagetable/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "vagetable", + "seen": [ + "pale yellow vagetable", + "medium oval vagetable", + "smooth oval vagetable", + "yellow food vagetable", + "yellow edible vagetable", + "edible yellow vagetable", + "hand-sized oval vagetable", + "vagetable with brown spots", + "smooth and patched vagetable", + "organic pale yellow vagetable", + "oval-shaped vagetable with spots", + "yellowish vagetable with rough patches" + ], + "unseen": [ + "brown-spotted oval vagetable", + "vagetable with dark brown marks", + "irregular pale yellow vagetable" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/069_vagetable/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/069_vagetable/base1.json new file mode 100644 index 00000000..6f637515 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/069_vagetable/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "vagetable", + "seen": [ + "yellow onion", + "round bulb onion", + "yellow onion bulb", + "medium yellow onion", + "light yellow vegetable", + "onion with pointed tip", + "medium-sized yellow bulb", + "palm-sized yellow round onion", + "yellow onion with tapered ends", + "medium onion with brown streaks", + "edible onion with smooth surface", + "light yellow onion with brown lines" + ], + "unseen": [ + "smooth round vegetable", + "smooth onion with papery skin", + "round onion with papery outer skin" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/069_vagetable/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/069_vagetable/base2.json new file mode 100644 index 00000000..c31f33ad --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/069_vagetable/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "vegetable", + "seen": [ + "dark green vegetable", + "long green vegetable", + "vegetable with smooth skin", + "dark green edible vegetable", + "slightly shiny green vegetable", + "green vegetable with a stem end", + "cucumber-shaped green vegetable", + "dark green cylindrical vegetable", + "vegetable with light green streaks", + "long vegetable with smooth texture", + "vegetable with small yellowish stem", + "medium green vegetable with rounded tips" + ], + "unseen": [ + "smooth green vegetable", + "vegetable with tapered ends", + "medium-sized green vegetable" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/069_vagetable/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/069_vagetable/base3.json new file mode 100644 index 00000000..2232113a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/069_vagetable/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "vagetable", + "seen": [ + "round tomato", + "fresh red tomato", + "bright red tomato", + "round shiny tomato", + "red tomato with green stem", + "bright red tomato with stem", + "smooth red vegetable tomato", + "smooth tomato with green top", + "glossy tomato with leafy top", + "small tomato with smooth skin", + "shiny tomato with green leafy top", + "vegetable tomato with green leaves" + ], + "unseen": [ + "shiny red tomato", + "red ball-shaped tomato", + "small palm-sized tomato" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/069_vagetable/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/069_vagetable/base4.json new file mode 100644 index 00000000..9d345756 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/069_vagetable/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "vagetable", + "seen": [ + "eggplant", + "purple eggplant", + "deep purple eggplant", + "medium-sized vegetable", + "food vegetable eggplant", + "eggplant with shiny skin", + "oval-shaped purple eggplant", + "hand-sized purple vegetable", + "smooth oval purple vegetable", + "green stem on purple eggplant", + "purple eggplant with green stem", + "dark purple vegetable with green top" + ], + "unseen": [ + "smooth oval eggplant", + "dark smooth-skinned eggplant", + "shiny eggplant with curved tip" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/069_vagetable/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/069_vagetable/base5.json new file mode 100644 index 00000000..030e1c89 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/069_vagetable/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "vagetable", + "seen": [ + "hand-sized corn cob", + "corn with green husk", + "green husk-covered corn", + "bright yellow rows on cob", + "corn cob with shiny kernels", + "green leafy covering on corn", + "corn with smooth yellow kernels", + "long green and yellow vegetable", + "corn with smooth rounded kernels", + "corn husk wrapping yellow kernels", + "vegetable with bright yellow rows", + "long vegetable with yellow kernels" + ], + "unseen": [ + "yellow corn", + "corn cob with leafy husks", + "natural plant with husks and cob" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/069_vagetable/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/069_vagetable/base6.json new file mode 100644 index 00000000..be4bfc61 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/069_vagetable/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "vagetable", + "seen": [ + "orange vagetable", + "carrot-shaped vagetable", + "cone-shaped orange vagetable", + "medium-sized orange vagetable", + "orange vagetable fits in palm", + "vagetable with green leafy top", + "bright orange carrot-like food", + "smooth orange carrot with leaves", + "orange body green leafy attachment", + "edible vagetable bright orange body", + "vagetable cone-shaped with leafy base", + "smooth orange food green rough leaves" + ], + "unseen": [ + "vagetable with rough green leaves", + "long orange vagetable with leafy end", + "solid carrot-like vagetable green leaves" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/070_paymentsign/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/070_paymentsign/base0.json new file mode 100644 index 00000000..b84a9b5e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/070_paymentsign/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "paymentsign", + "seen": [ + "beige paymentsign", + "QR code paymentsign", + "smooth beige paymentsign", + "paymentsign showing QR code label", + "paymentsign resting on angled base", + "paymentsign with blue and white text", + "small paymentsign for mobile payments", + "beige paymentsign with printed details", + "rectangular paymentsign with colorful top", + "compact paymentsign for scanning payments", + "stand-up paymentsign with rectangular top", + "paymentsign with tilted rectangular panel" + ], + "unseen": [ + "tabletop paymentsign", + "paymentsign with QR code", + "paymentsign with angled stand" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/070_paymentsign/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/070_paymentsign/base1.json new file mode 100644 index 00000000..5733a2c0 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/070_paymentsign/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "paymentsign", + "seen": [ + "tabletop paymentsign", + "small glossy paymentsign", + "blue and white paymentsign", + "smooth surface paymentsign", + "small paymentsign with stand", + "rectangular plastic paymentsign", + "paymentsign for scanning QR codes", + "paymentsign with printed QR codes", + "paymentsign with glossy blue panel", + "paymentsign with angled black stand", + "blue and white rectangular paymentsign", + "rectangular plastic paymentsign with codes" + ], + "unseen": [ + "QR code paymentsign", + "paymentsign with black metal stand", + "paymentsign with angled support bars" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/070_paymentsign/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/070_paymentsign/base2.json new file mode 100644 index 00000000..e9f2d305 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/070_paymentsign/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "paymentsign", + "seen": [ + "paymentsign", + "gray top with QR code", + "tabletop-sized payment sign", + "gray slanted panel with text", + "payment sign with green logo", + "payment sign with angled face", + "triangular stand with QR code", + "small black stand for payments", + "green payment logo on gray top", + "black base with tilted gray panel", + "small payment stand with black base", + "slanted gray panel for QR code scan" + ], + "unseen": [ + "QR code payment sign", + "black triangular stand", + "black stand with angled display" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/070_paymentsign/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/070_paymentsign/base3.json new file mode 100644 index 00000000..b062703a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/070_paymentsign/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "paymentsign", + "seen": [ + "paymentsign", + "white stand sign", + "tabletop paymentsign", + "plastic smooth paymentsign", + "paymentsign with QR code area", + "yellow panel white paymentsign", + "sign with yellow payment section", + "yellow and white payment display", + "rectangular paymentsign with stand", + "white paymentsign with raised base", + "smooth plastic paymentsign with QR code", + "white rectangular paymentsign with yellow details" + ], + "unseen": [ + "sign for payment processing", + "medium paymentsign with QR section", + "paymentsign with smooth base stand" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/071_can/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/071_can/base0.json new file mode 100644 index 00000000..feb45f20 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/071_can/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "can", + "seen": [ + "white can", + "medium metal can", + "can with gold top lid", + "can with red-orange text", + "can with sealed gold top", + "cylindrical white glossy can", + "red-orange text on white can", + "white can with 3D RADY label", + "smooth white can with accents", + "small white can with branding", + "beverage can with printed label", + "can with rounded cylindrical shape" + ], + "unseen": [ + "metal can fits easily in hand", + "can with shiny smooth surface", + "lightweight cylindrical metal can" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/071_can/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/071_can/base1.json new file mode 100644 index 00000000..5dd4113a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/071_can/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "can", + "seen": [ + "red can", + "metallic soda can", + "smooth red soda can", + "palm-sized drink can", + "compact metal drink can", + "gold patterned beverage can", + "gold-decorated aluminum can", + "red and gold cylindrical can", + "bright red drink container can", + "red can with gold illustrations", + "metallic can with rounded edges", + "red beverage can with curved top" + ], + "unseen": [ + "small aluminum can", + "gold and red soda can", + "can with gold logo design" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/071_can/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/071_can/base2.json new file mode 100644 index 00000000..61543f55 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/071_can/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "can", + "seen": [ + "red green can", + "red green beverage can", + "can with red green label", + "metal can with pull-tab top", + "green red metallic soda can", + "drink can with bright colors", + "soda can with lime illustration", + "metallic drink can with pull-tab", + "small soda can with lime graphics", + "beverage can with red green graphics", + "colorful can with lime and orange design", + "smooth cylindrical can with lime pattern" + ], + "unseen": [ + "cylindrical can with bright colors", + "hand-sized can with metallic finish", + "cylindrical can with smooth surface" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/071_can/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/071_can/base3.json new file mode 100644 index 00000000..21feed6d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/071_can/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "can", + "seen": [ + "Coca-Cola can", + "Coca-Cola soda can", + "metallic red beverage can", + "small aluminum red drink can", + "red can with white lettering", + "hand-sized red can for drinks", + "red can with white Coca-Cola logo", + "smooth red can with silver bottom", + "red cylindrical can with silver top", + "cylindrical red drink can with logo", + "red can with smooth metallic texture", + "rounded red Coca-Cola soda container" + ], + "unseen": [ + "red can", + "small red soda can", + "palm-sized red metal Coca-Cola can" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/071_can/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/071_can/base5.json new file mode 100644 index 00000000..adad4fea --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/071_can/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "can", + "seen": [ + "brown can", + "rounded metallic can", + "smooth light brown can", + "hand-sized metallic can", + "white patterned brown metal can", + "drink can with light brown shade", + "small cylindrical drink container", + "small light brown cylindrical can", + "light brown can with smooth texture", + "smooth brown can with rounded edges", + "metal can with printed white graphics", + "metal can with white decorative designs" + ], + "unseen": [ + "light brown cylindrical can", + "rounded can with smooth printed surface", + "cylindrical brown drink can with white markings" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/071_can/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/071_can/base6.json new file mode 100644 index 00000000..8131f132 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/071_can/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "can", + "seen": [ + "smooth red can", + "cylindrical metal can", + "medium-sized soda can", + "metal can with red base", + "red and white beverage can", + "can with flat silver bottom", + "red drink can with white lid", + "red can with decorative label", + "red shiny can with green label", + "red soda can with green accents", + "drink can with smooth red finish", + "cylindrical can with flat surfaces" + ], + "unseen": [ + "red can", + "can with red body", + "red can with white top" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/072_electronicscale/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/072_electronicscale/base0.json new file mode 100644 index 00000000..aa3a9c4b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/072_electronicscale/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "electronicscale", + "seen": [ + "white electronicscale", + "compact electronicscale", + "handheld electronicscale", + "electronicscale portable", + "electronicscale red accents", + "electronicscale digital display", + "electronicscale white main body", + "electronicscale smooth and glossy", + "electronicscale small and rectangular", + "white electronicscale with red buttons", + "electronicscale black and white design", + "electronicscale with round weight platform" + ], + "unseen": [ + "electronicscale flat base and stand", + "electronicscale with circular platform", + "electronicscale with black digital screen" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/072_electronicscale/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/072_electronicscale/base1.json new file mode 100644 index 00000000..2c7fee56 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/072_electronicscale/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "electronicscale", + "seen": [ + "white electronicscale", + "compact electronicscale", + "white body electronicscale", + "electronicscale with black pad", + "small rectangular electronicscale", + "electronicscale with metallic dial", + "electronicscale with visible display", + "electronicscale for weighing objects", + "electronicscale with circular gold ring", + "electronicscale with clear digital screen", + "electronicscale with white plastic casing", + "electronicscale with flat rectangular base" + ], + "unseen": [ + "black screen electronicscale", + "electronicscale featuring gold accents", + "handheld electronicscale with smooth finish" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/072_electronicscale/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/072_electronicscale/base2.json new file mode 100644 index 00000000..898faa79 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/072_electronicscale/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "electronicscale", + "seen": [ + "white electronicscale", + "beige plate electronicscale", + "white base with black screen", + "smooth small electronicscale", + "black display electronicscale", + "small electronic weighing scale", + "digital electronicscale with display", + "electronicscale with rectangular base", + "electronicscale with circular beige dish", + "lightweight electronicscale for measurements", + "compact electronicscale with digital display", + "electronicscale with circular weighing platform" + ], + "unseen": [ + "electronicscale with beige plate", + "electronicscale with green screen", + "table-top electronicscale with plate" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/072_electronicscale/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/072_electronicscale/base5.json new file mode 100644 index 00000000..c4085b25 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/072_electronicscale/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "electronicscale", + "seen": [ + "compact white electronicscale", + "black and white electronicscale", + "electronicscale with rubber feet", + "small rectangular electronicscale", + "electronicscale with smooth surface", + "electronicscale with digital screen", + "electronicscale with silver buttons", + "electronicscale with a glass display", + "electronicscale with small LCD screen", + "electronicscale with black weighing pad", + "electronicscale with black control panel", + "modern electronicscale with durable design" + ], + "unseen": [ + "electronicscale", + "scale with rounded edges", + "lightweight electronicscale for measuring weight" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/072_electronicscale/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/072_electronicscale/base6.json new file mode 100644 index 00000000..87794dd9 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/072_electronicscale/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "electronicscale", + "seen": [ + "electronicscale", + "white electronicscale", + "compact electronicscale", + "handheld electronicscale", + "white body electronicscale", + "lightweight electronicscale", + "small weighing electronicscale", + "electronicscale with smooth finish", + "electronicscale with digital screen", + "medium electronicscale with gold top", + "electronicscale for measuring weights", + "electronicscale with black display screen" + ], + "unseen": [ + "electronicscale with rectangular base", + "gold circular platform electronicscale", + "electronicscale with gold round platform" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/073_rubikscube/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/073_rubikscube/base0.json new file mode 100644 index 00000000..b8325335 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/073_rubikscube/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "rubikscube", + "seen": [ + "rubikscube", + "colorful cube", + "small puzzle cube", + "plastic rubikscube", + "black-bordered rubikscube", + "rubikscube made of square tiles", + "smooth cube with twisting blocks", + "rubikscube with six colored faces", + "rubikscube with red blue green tiles", + "rubikscube with matching color blocks", + "rubikscube with yellow and orange squares", + "rubikscube with rotating rows and columns" + ], + "unseen": [ + "handheld rubikscube", + "multi-colored rubikscube", + "rubikscube for twisting and turning" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/073_rubikscube/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/073_rubikscube/base1.json new file mode 100644 index 00000000..019d4023 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/073_rubikscube/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "rubikscube", + "seen": [ + "rubikscube", + "multi-colored cube", + "small plastic rubikscube", + "rubikscube with smooth square tiles", + "rubikscube with bright primary colors", + "rubikscube showing red and white squares", + "puzzle rubikscube with 3x3 grid per side", + "smooth rubikscube with black frame edges", + "plastic rubikscube with six colored sides", + "multi-sided rubikscube with rotating pieces", + "colorful rubikscube for twisting and solving", + "rubikscube featuring white, green, orange tiles" + ], + "unseen": [ + "red, blue, green, yellow rubikscube", + "toy cube with orange and blue squares", + "hand-sized rubikscube with colored squares" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/073_rubikscube/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/073_rubikscube/base2.json new file mode 100644 index 00000000..cd910475 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/073_rubikscube/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "rubikscube", + "seen": [ + "rubikscube", + "colorful rubikscube", + "rubikscube cube shape", + "yellow-blue rubikscube", + "small square rubikscube", + "multicolored puzzle cube", + "rubikscube with orange sides", + "rubikscube plastic and smooth", + "rubikscube with rotating faces", + "rubikscube colorful and compact", + "rubikscube with green center tiles", + "rubikscube with smooth plastic tiles" + ], + "unseen": [ + "handheld rubikscube", + "rubikscube with yellow top", + "rubikscube for puzzle solving" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/074_displaystand/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/074_displaystand/base0.json new file mode 100644 index 00000000..d2d38254 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/074_displaystand/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "displaystand", + "seen": [ + "plastic displaystand", + "rectangular displaystand", + "smooth black displaystand", + "medium-size black displaystand", + "black stand for display purposes", + "angled black smooth displaystand", + "hollow black plastic displaystand", + "black displaystand for holding items", + "medium-sized rectangular displaystand", + "rectangular wedge-shaped displaystand", + "sturdy black rectangular displaystand", + "plastic displaystand with hollow interior" + ], + "unseen": [ + "black displaystand", + "displaystand with angled sides", + "compact black hollow displaystand" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/074_displaystand/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/074_displaystand/base1.json new file mode 100644 index 00000000..b7c5ea2d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/074_displaystand/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "displaystand", + "seen": [ + "black displaystand", + "plastic displaystand", + "rectangular displaystand", + "medium black displaystand", + "hollow black displaystand", + "displaystand with angled sides", + "rectangular plastic displaystand", + "smooth textured black displaystand", + "shiny black rectangular displaystand", + "displaystand with rectangular hollow space", + "black shiny displaystand with smooth texture", + "medium smooth black displaystand for displays" + ], + "unseen": [ + "tabletop-sized displaystand", + "displaystand with hollow center", + "medium displaystand with smooth finish" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/074_displaystand/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/074_displaystand/base2.json new file mode 100644 index 00000000..015d3bac --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/074_displaystand/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "displaystand", + "seen": [ + "metal displaystand", + "medium displaystand", + "angled displaystand", + "rectangular displaystand", + "smooth brown displaystand", + "displaystand with slanted sides", + "rectangular brown stand for items", + "smooth dark metallic displaystand", + "metallic brown displaystand with cutout", + "displaystand with rectangular hollow base", + "brown displaystand with rectangular opening", + "medium-sized displaystand with smooth finish" + ], + "unseen": [ + "dark brown displaystand", + "medium brown displaystand with slot", + "angled rectangular metal displaystand" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/074_displaystand/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/074_displaystand/base3.json new file mode 100644 index 00000000..97de61e4 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/074_displaystand/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "displaystand", + "seen": [ + "displaystand frame", + "black smooth stand", + "black rectangular frame", + "simple black displaystand", + "rectangular black stand frame", + "displaystand for holding items", + "smooth black hollow displaystand", + "compact black displaystand frame", + "medium black plastic displaystand", + "plastic black rectangular displaystand", + "black displaystand for showcasing things", + "lightweight black rectangular displaystand" + ], + "unseen": [ + "hollow rectangular black displaystand", + "medium-sized hollow black displaystand", + "black frame displaystand with open sides" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/074_displaystand/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/074_displaystand/base4.json new file mode 100644 index 00000000..6ff84194 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/074_displaystand/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "displaystand", + "seen": [ + "black displaystand", + "matte black displaystand", + "angled black displaystand", + "plastic matte displaystand", + "hand-held black displaystand", + "displaystand for holding items", + "displaystand with smooth finish", + "displaystand with angled support", + "medium black angled displaystand", + "compact angled black displaystand", + "smooth black rectangular displaystand", + "plastic black displaystand for display" + ], + "unseen": [ + "rectangular displaystand", + "medium-sized displaystand", + "rectangular base displaystand" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/075_bread/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/075_bread/base0.json new file mode 100644 index 00000000..c8337a9f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/075_bread/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bread", + "seen": [ + "small bread", + "soft golden bread", + "soft doughy bread", + "golden brown bread", + "braided bread loaf", + "bread with golden crust", + "small golden baked bread", + "smooth crusty small bread", + "round bread with braided top", + "round bread with crusty texture", + "golden bread loaf with soft inside", + "golden brown bread with braided design" + ], + "unseen": [ + "round braided loaf", + "braided top round bread", + "palm-sized braided golden bread" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/075_bread/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/075_bread/base1.json new file mode 100644 index 00000000..c1b642fa --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/075_bread/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bread", + "seen": [ + "soft bread", + "golden bread", + "cuboid bread", + "golden bread loaf", + "square-shaped bread", + "doughy and soft bread", + "hand-sized bread block", + "light golden brown bread", + "bread with rough texture", + "bread loaf with uneven edges", + "bread loaf with bumpy surface", + "bread block with light patches" + ], + "unseen": [ + "rectangular bread loaf", + "bread with golden crust", + "bread loaf with soft surface" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/075_bread/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/075_bread/base2.json new file mode 100644 index 00000000..98a42363 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/075_bread/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bread", + "seen": [ + "bread loaf", + "golden bread", + "loaf of bread", + "bread with rounded top", + "brown-topped bread loaf", + "bread with soft golden top", + "medium-sized rectangular bread", + "white and golden rectangular bread", + "fluffy bread with rectangular shape", + "block-shaped bread with rounded edges", + "golden crust bread with white interior", + "rectangular bread loaf with fluffy texture" + ], + "unseen": [ + "soft block of bread", + "bread loaf with golden baked top", + "medium bread loaf with soft crust" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/075_bread/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/075_bread/base3.json new file mode 100644 index 00000000..4a79bf22 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/075_bread/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bread", + "seen": [ + "brown loaf", + "soft brown bread", + "golden bread loaf", + "bread with split top", + "hand-sized oval loaf", + "small golden brown bread", + "small loaf with curved top", + "bread with warm golden color", + "light brown bread with ridges", + "bread with golden-brown crust", + "smooth loaf with rounded edges", + "light brown loaf with smooth sides" + ], + "unseen": [ + "oval bread", + "rounded oval bread", + "oval-shaped soft bread" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/075_bread/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/075_bread/base4.json new file mode 100644 index 00000000..102e1cba --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/075_bread/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bread", + "seen": [ + "soft loaf bread", + "golden brown loaf", + "soft medium bread", + "medium-sized bread", + "bread with golden crust", + "golden brown loaf bread", + "soft golden medium bread", + "round-topped golden bread", + "medium-sized golden bread", + "round-topped loaf of bread", + "golden loaf with soft crust", + "golden bread with crusty surface" + ], + "unseen": [ + "medium bread loaf", + "rounded loaf bread", + "loaf-shaped brown bread" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/075_bread/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/075_bread/base5.json new file mode 100644 index 00000000..c85e74eb --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/075_bread/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bread", + "seen": [ + "soft square bread", + "golden brown bread", + "fluffy baked bread", + "golden baked bread", + "rounded square loaf", + "bread with soft texture", + "bread with rounded edges", + "medium-sized bread piece", + "light brown porous bread", + "square-shaped bread piece", + "sponge-textured golden bread", + "medium bread with rounded sides" + ], + "unseen": [ + "spongy golden brown bread", + "soft bread with golden edges", + "square bread with light spots" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/075_bread/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/075_bread/base6.json new file mode 100644 index 00000000..1f8c7622 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/075_bread/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bread", + "seen": [ + "golden bread", + "brown bread loaf", + "slice-shaped bread", + "square loaf of bread", + "soft golden baked bread", + "bread with rounded corners", + "golden crusted bread slice", + "bread with firm crust edges", + "bread loaf with square shape", + "rectangular bread soft inside", + "golden brown loaf-shaped bread", + "bread with golden surface color" + ], + "unseen": [ + "small loaf-shaped bread", + "bread loaf medium-sized", + "soft rectangular bread loaf" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/076_breadbasket/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/076_breadbasket/base0.json new file mode 100644 index 00000000..208081de --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/076_breadbasket/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "breadbasket", + "seen": [ + "beige breadbasket", + "rectangular breadbasket", + "plastic mesh breadbasket", + "textured beige breadbasket", + "woven light beige breadbasket", + "breadbasket for bread storage", + "medium rectangular breadbasket", + "light and textured breadbasket", + "breadbasket with textured sides", + "plastic breadbasket with mesh sides", + "beige breadbasket for holding bread", + "light beige rectangular breadbasket" + ], + "unseen": [ + "woven shallow breadbasket", + "breadbasket with woven pattern", + "shallow rectangular breadbasket" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/076_breadbasket/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/076_breadbasket/base1.json new file mode 100644 index 00000000..044383b4 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/076_breadbasket/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "breadbasket", + "seen": [ + "beige breadbasket", + "white oval breadbasket", + "oval-shaped breadbasket", + "beige plastic breadbasket", + "smooth textured breadbasket", + "medium oval basket for bread", + "basket for bread beige plastic", + "basket for carrying bread beige", + "beige basket with white grid design", + "medium plastic breadbasket oval shape", + "oval perforated breadbasket white edges", + "medium breadbasket with perforated sides" + ], + "unseen": [ + "white basket with beige bottom", + "breadbasket with hollowed white walls", + "breadbasket with smooth bottom grid sides" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/076_breadbasket/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/076_breadbasket/base2.json new file mode 100644 index 00000000..e9ff4862 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/076_breadbasket/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "breadbasket", + "seen": [ + "oval breadbasket", + "oval basket for bread", + "light brown breadbasket", + "medium oval breadbasket", + "woven plastic breadbasket", + "two-hand-sized breadbasket", + "smooth-textured breadbasket", + "breadbasket with curved base", + "beige basket for holding bread", + "synthetic material breadbasket", + "basket with light brown weaving", + "breadbasket for small food items" + ], + "unseen": [ + "beige breadbasket", + "medium-sized breadbasket", + "breadbasket with ridged texture" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/076_breadbasket/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/076_breadbasket/base3.json new file mode 100644 index 00000000..d8545988 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/076_breadbasket/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "breadbasket", + "seen": [ + "brown breadbasket", + "medium woven breadbasket", + "light brown basket for bread", + "woven rectangular breadbasket", + "breadbasket for holding bread", + "breadbasket with rounded edges", + "light brown wicker breadbasket", + "smooth textured brown breadbasket", + "medium-sized rectangular breadbasket", + "breadbasket with smooth woven texture", + "breadbasket with wicker weave pattern", + "brown breadbasket with rounded corners" + ], + "unseen": [ + "rectangular breadbasket", + "medium light brown breadbasket", + "rectangular light brown breadbasket" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/076_breadbasket/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/076_breadbasket/base4.json new file mode 100644 index 00000000..5a6e539d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/076_breadbasket/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "breadbasket", + "seen": [ + "round breadbasket", + "medium breadbasket", + "wooden breadbasket", + "breadbasket for dough", + "palm-sized breadbasket", + "breadbasket for baking", + "smooth beige breadbasket", + "spiral ridged breadbasket", + "breadbasket with flat base", + "breadbasket with spiral lines", + "breadbasket made of woven wood", + "breadbasket shaped like a bowl" + ], + "unseen": [ + "beige breadbasket", + "round beige breadbasket", + "light wooden breadbasket" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/077_phone/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/077_phone/base0.json new file mode 100644 index 00000000..708e6b80 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/077_phone/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "phone", + "seen": [ + "flat phone", + "compact phone", + "glass-faced phone", + "lightweight phone", + "smooth-screen phone", + "black and white phone", + "phone with touchscreen", + "sleek rectangular phone", + "phone for calls and apps", + "phone with digital display", + "phone with buttons and screen", + "palm-sized black and white phone" + ], + "unseen": [ + "white-sided phone", + "rectangular phone", + "phone with rounded edges" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/077_phone/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/077_phone/base1.json new file mode 100644 index 00000000..8a40b05a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/077_phone/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "phone", + "seen": [ + "sleek phone", + "silver phone", + "palm-sized phone", + "rectangular phone", + "phone with metallic body", + "phone with black touchscreen", + "phone with smooth back panel", + "phone with smooth glass front", + "silver phone with black screen", + "compact phone with flat design", + "phone with screen and side buttons", + "metallic phone with glass screen front" + ], + "unseen": [ + "modern phone", + "handheld phone with silver frame", + "rectangular silver phone with rounded edges" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/077_phone/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/077_phone/base2.json new file mode 100644 index 00000000..3a277f84 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/077_phone/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "phone", + "seen": [ + "black phone", + "silver-edged phone", + "slim palm-sized phone", + "glass and metal smartphone", + "sleek black-faced smartphone", + "phone with smooth glossy screen", + "compact phone with shiny texture", + "phone with polished silver border", + "black front phone with silver sides", + "rectangle-shaped communication phone", + "rectangular phone with rounded corners", + "phone featuring glass and silver frame" + ], + "unseen": [ + "phone with flat black screen", + "smartphone with black display", + "rectangular phone with curved edges" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/077_phone/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/077_phone/base3.json new file mode 100644 index 00000000..f484d44e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/077_phone/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "phone", + "seen": [ + "phone", + "rectangular phone", + "smooth glass phone", + "metal and glass smartphone", + "medium-sized handheld phone", + "brown phone with gray details", + "phone with camera lens at back", + "phone with printed back design", + "brown and gray multimedia phone", + "dark brown phone with smooth surface", + "phone with unique printed back pattern", + "rectangular glossy phone with dark color" + ], + "unseen": [ + "dark phone with shiny finish", + "flat-edged rectangular phone", + "phone with flat rectangular back" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/077_phone/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/077_phone/base4.json new file mode 100644 index 00000000..fe654406 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/077_phone/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "phone", + "seen": [ + "black phone", + "sleek black phone", + "smooth black phone", + "thin metallic phone", + "glass-front black phone", + "multi-camera black phone", + "compact rectangular phone", + "black shiny rectangle phone", + "metal and glass black phone", + "rectangular glossy black phone", + "thin phone with smooth texture", + "phone with glass front and back" + ], + "unseen": [ + "phone with rounded edges", + "phone with shiny black back", + "palm-sized phone with glossy finish" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/078_phonestand/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/078_phonestand/base0.json new file mode 100644 index 00000000..6087b204 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/078_phonestand/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "phonestand", + "seen": [ + "black phonestand", + "adjustable phonestand", + "medium-sized phonestand", + "gray round base phonestand", + "phonestand for holding phones", + "two-part adjustable phonestand", + "compact phonestand for desk use", + "plastic phonestand with metal support", + "phonestand with smooth adjustable arm", + "phonestand with textured black platform", + "phonestand with black rectangular holder", + "adjustable phonestand with sturdy design" + ], + "unseen": [ + "rectangular phone holder on stand", + "phonestand with circular gray base", + "lightweight black and gray phonestand" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/078_phonestand/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/078_phonestand/base1.json new file mode 100644 index 00000000..4605275f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/078_phonestand/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "phonestand", + "seen": [ + "green phone rack", + "compact phonestand", + "green plastic holder", + "flat base phonestand", + "phonestand with silver bar", + "angled green phone support", + "phonestand with green hooks", + "metal adjustable phone clamp", + "dark green holder with clamp", + "small green and silver stand", + "phonestand with smooth texture", + "compact adjustable phone holder" + ], + "unseen": [ + "dark green phonestand", + "silver arm phonestand", + "adjustable phone holder" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/078_phonestand/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/078_phonestand/base2.json new file mode 100644 index 00000000..e60bfc57 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/078_phonestand/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "phonestand", + "seen": [ + "phonestand with slot", + "light brown phonestand", + "small plastic phonestand", + "light brown phone holder", + "angled holder for phones", + "smooth brown phone holder", + "compact plastic phonestand", + "phonestand with curved edges", + "phonestand for holding phones", + "phonestand with smooth texture", + "curved phonestand with phone slot", + "phone holder with cutout underneath" + ], + "unseen": [ + "angled brown phonestand", + "brown angled plastic phonestand", + "L-shaped smooth brown phonestand" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/078_phonestand/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/078_phonestand/base3.json new file mode 100644 index 00000000..482ad303 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/078_phonestand/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "phonestand", + "seen": [ + "adjustable phone holder", + "flat brown phone holder", + "brown and black phonestand", + "brown circular phone holder", + "plastic and metal phonestand", + "phonestand with matte finish", + "phonestand with circular back", + "phonestand for holding phones", + "phonestand with angular black base", + "smooth phonestand with matte finish", + "phonestand with adjustable black bar", + "medium-sized phonestand with smooth texture" + ], + "unseen": [ + "medium phonestand", + "black stand with brown plate", + "black stand brown holder phonestand" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/078_phonestand/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/078_phonestand/base4.json new file mode 100644 index 00000000..9f65cc91 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/078_phonestand/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "phonestand", + "seen": [ + "silver-gray phonestand", + "silver-gray base and arm", + "smooth surface phonestand", + "phone holder on narrow arm", + "polished silver phonestand", + "phonestand with flexible arm", + "phonestand with circular base", + "rectangle-shaped phone holder", + "phonestand with adjustable arm", + "phonestand with smooth texture", + "compact phonestand with sturdy base", + "adjustable silver-gray phone holder" + ], + "unseen": [ + "small phone holder stand", + "metal-and-plastic phonestand", + "hand-sized adjustable phonestand" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/078_phonestand/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/078_phonestand/base5.json new file mode 100644 index 00000000..ebc0af54 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/078_phonestand/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "phonestand", + "seen": [ + "metal phonestand", + "dark gray phonestand", + "angled phone holder stand", + "adjustable angled phonestand", + "smooth metallic phone holder", + "small rectangular phone stand", + "metallic dark gray phone stand", + "phone stand with smooth surface", + "angled holder on rectangular base", + "dark gray rectangular phone holder", + "metal phone stand with smooth base", + "rectangular phonestand with angled holder" + ], + "unseen": [ + "small phone holder", + "phone holder with angled top", + "compact dark gray phone stand" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/078_phonestand/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/078_phonestand/base6.json new file mode 100644 index 00000000..7c3e4f8a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/078_phonestand/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "phonestand", + "seen": [ + "gray phonestand", + "gray stand for phones", + "adjustable phone stand", + "gray and blue phone holder", + "medium plastic phone stand", + "phone stand with flat base", + "rectangular phonestand base", + "phonestand with tilting arm", + "plastic and metal phonestand", + "phonestand with metal hinges", + "phonestand with adjustable arm", + "phonestand with smooth surface" + ], + "unseen": [ + "light gray phonestand", + "medium adjustable phonestand", + "phonestand with smooth matte finish" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/079_remotecontrol/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/079_remotecontrol/base0.json new file mode 100644 index 00000000..0bba4161 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/079_remotecontrol/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "remote control", + "seen": [ + "slim remote control", + "black plastic remote control", + "remote control for electronics", + "remote control with smooth body", + "rectangular black remote control", + "remote control with number keypad", + "remote control for TVs and devices", + "remote control with colored buttons", + "remote control with textured buttons", + "remote control with rubberized buttons", + "remote control with red and yellow buttons", + "remote control with long rectangular shape" + ], + "unseen": [ + "black remote control", + "small handheld remote control", + "remote control with circular pad" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/079_remotecontrol/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/079_remotecontrol/base1.json new file mode 100644 index 00000000..ea8840c3 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/079_remotecontrol/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "remotecontrol", + "seen": [ + "white remotecontrol", + "smooth remotecontrol", + "small white remotecontrol", + "remotecontrol for electronics", + "handheld plastic remotecontrol", + "flat rectangular remotecontrol", + "white rectangular remotecontrol", + "lightweight white remotecontrol", + "remotecontrol with raised buttons", + "remotecontrol with smooth surface", + "remotecontrol for controlling devices", + "remotecontrol with green and red buttons" + ], + "unseen": [ + "rectangular remotecontrol", + "plastic white remotecontrol", + "remotecontrol with colorful buttons" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/079_remotecontrol/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/079_remotecontrol/base2.json new file mode 100644 index 00000000..9f88ec2d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/079_remotecontrol/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "remotecontrol", + "seen": [ + "black remotecontrol", + "compact remotecontrol", + "slim black remotecontrol", + "remotecontrol with buttons", + "black plastic remotecontrol", + "small handheld remotecontrol", + "lightweight black remotecontrol", + "remotecontrol with smooth surface", + "remotecontrol with tactile buttons", + "remotecontrol with colorful buttons", + "remotecontrol for TV and electronics", + "remotecontrol with yellow and blue buttons" + ], + "unseen": [ + "rectangular remotecontrol", + "remotecontrol with rounded edges", + "remotecontrol with a curved shape" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/079_remotecontrol/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/079_remotecontrol/base3.json new file mode 100644 index 00000000..4e224a9a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/079_remotecontrol/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "remotecontrol", + "seen": [ + "remotecontrol", + "black remotecontrol", + "sleek remotecontrol", + "flat dark remotecontrol", + "hand-sized remotecontrol", + "remotecontrol with buttons", + "small rectangular remotecontrol", + "remotecontrol with slanted sides", + "remotecontrol with numeric keypad", + "remotecontrol with smooth surface", + "rectangular remotecontrol with red button", + "remotecontrol with circular navigation pad" + ], + "unseen": [ + "remotecontrol designed for TV", + "dark gray plastic remotecontrol", + "remotecontrol with button layout" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/079_remotecontrol/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/079_remotecontrol/base4.json new file mode 100644 index 00000000..ba7a0792 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/079_remotecontrol/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "remotecontrol", + "seen": [ + "white remotecontrol", + "smooth remotecontrol", + "plastic remotecontrol", + "remotecontrol with slim design", + "white rectangular remotecontrol", + "remotecontrol with tapered ends", + "remotecontrol with black buttons", + "remotecontrol with raised buttons", + "remotecontrol with plastic casing", + "remotecontrol for electronic devices", + "remotecontrol with red button cluster", + "white remotecontrol with black top slot" + ], + "unseen": [ + "handheld remotecontrol", + "rectangular remotecontrol", + "remotecontrol with red and black buttons" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/079_remotecontrol/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/079_remotecontrol/base5.json new file mode 100644 index 00000000..f143b47c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/079_remotecontrol/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "remotecontrol", + "seen": [ + "white body remotecontrol", + "remotecontrol with buttons", + "black button remotecontrol", + "small handheld remotecontrol", + "white and black remotecontrol", + "wireless control remotecontrol", + "remotecontrol with curved edges", + "remotecontrol with many buttons", + "remotecontrol with smooth texture", + "remotecontrol with rectangular shape", + "palm-sized rectangular remotecontrol", + "remotecontrol for televisions and electronics" + ], + "unseen": [ + "smooth plastic remotecontrol", + "remotecontrol with top black region", + "remotecontrol for controlling devices" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/079_remotecontrol/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/079_remotecontrol/base6.json new file mode 100644 index 00000000..d173f3ed --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/079_remotecontrol/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "remotecontrol", + "seen": [ + "black remotecontrol", + "black plastic remotecontrol", + "handheld gray remotecontrol", + "black tapered remotecontrol", + "small plastic remotecontrol", + "black and gray remotecontrol", + "remotecontrol with gray casing", + "remotecontrol with red buttons", + "remotecontrol with raised buttons", + "remotecontrol with red power button", + "remotecontrol with circular navigation dial", + "rectangular remotecontrol with multiple buttons" + ], + "unseen": [ + "remotecontrol with smooth surface", + "gray remotecontrol with black sides", + "remotecontrol for controlling devices" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/080_pillbottle/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/080_pillbottle/base1.json new file mode 100644 index 00000000..2fc5cd03 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/080_pillbottle/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "smooth plastic bottle", + "bottle with bright orange top", + "white bottle with orange label", + "bottle with large orange label", + "white and orange storage bottle", + "round bottle with orange screw-top", + "small bottle with ridged orange cap", + "palm-sized bottle with orange cover", + "white body bottle with textured cap", + "white bottle with ridged orange lid", + "plastic bottle with orange screw lid", + "compact plastic bottle with screw cap" + ], + "unseen": [ + "orange capped bottle", + "white cylindrical bottle", + "hand-sized orange capped bottle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/080_pillbottle/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/080_pillbottle/base2.json new file mode 100644 index 00000000..886f875d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/080_pillbottle/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "white bottle", + "smooth white bottle", + "small plastic bottle", + "white cylindrical bottle", + "bottle with white screw cap", + "plastic bottle with ridged cap", + "white bottle with screw-on lid", + "white bottle for pills or liquids", + "hand-held cylindrical white bottle", + "plastic bottle with labeled surface", + "bottle with smooth body and ridged cap", + "medium-sized white cylindrical container" + ], + "unseen": [ + "medium-sized bottle", + "white bottle for storing items", + "cylindrical bottle with ribbed cap" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/080_pillbottle/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/080_pillbottle/base3.json new file mode 100644 index 00000000..f5b2f732 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/080_pillbottle/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "bottle with white cap", + "flat base orange bottle", + "white cap on orange bottle", + "medium-sized pharmacy bottle", + "smooth plastic orange bottle", + "medicine bottle with screw cap", + "orange bottle for pills or liquid", + "orange cylindrical bottle with label", + "orange bottle with blue stripe label", + "medium orange bottle with flat bottom", + "orange cylindrical bottle for medicine", + "bottle featuring blue horizontal stripe" + ], + "unseen": [ + "orange bottle", + "bottle with printed white label", + "plastic orange bottle with white lid" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/080_pillbottle/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/080_pillbottle/base4.json new file mode 100644 index 00000000..bd32e054 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/080_pillbottle/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "small brown bottle", + "rounded small bottle", + "lightweight plastic bottle", + "brown bottle with white cap", + "smooth-textured brown bottle", + "plastic bottle with brown body", + "white-capped cylindrical bottle", + "bottle with white label and cap", + "brown plastic cylindrical bottle", + "brown bottle with wide white label", + "cylindrical bottle with brown surface", + "small brown bottle with white accents" + ], + "unseen": [ + "bottle with white screw-on cap", + "bottle with smooth brown texture", + "bottle capped with textured white lid" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/080_pillbottle/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/080_pillbottle/base5.json new file mode 100644 index 00000000..3e864cbc --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/080_pillbottle/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "white bottle", + "teal and white plastic bottle", + "plastic bottle with teal bands", + "bottle with teal top and bottom", + "rounded bottle with small white cap", + "plastic bottle with ridged white cap", + "white bottle with teal rounded edges", + "smooth plastic bottle with teal accents", + "smooth cylindrical bottle with teal ends", + "teal-bottom bottle with cylindrical body", + "cylindrical bottle with ridged white lid", + "compact bottle with teal and white design" + ], + "unseen": [ + "white bottle with teal ends", + "hand-sized bottle with screw cap", + "rounded white bottle with teal base" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/081_playingcards/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/081_playingcards/base0.json new file mode 100644 index 00000000..81760443 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/081_playingcards/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "playingcards", + "seen": [ + "box of playingcards", + "blue playingcards box", + "smooth playingcards holder", + "small playingcards container", + "card box with golden designs", + "blue carton for playingcards", + "blue pack holding playingcards", + "palm-sized playingcards carton", + "playingcards pack with blue top", + "rectangular blue playingcards case", + "playingcards box with white symbols", + "blue and white playingcards packaging" + ], + "unseen": [ + "small packet of playingcards", + "rectangular playingcards pack", + "rectangular playingcards case with patterns" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/081_playingcards/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/081_playingcards/base1.json new file mode 100644 index 00000000..d537d7de --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/081_playingcards/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "playingcards", + "seen": [ + "red playingcards box", + "box for playingcards", + "small playingcards pack", + "smooth playingcards pack", + "playingcards storage box", + "rectangular playingcards case", + "compact red playingcards pack", + "yellow and red playingcards box", + "red and yellow playingcards case", + "hand-sized playingcards container", + "glossy rectangular playingcards box", + "playingcards box with white accents" + ], + "unseen": [ + "small box for playingcards", + "playingcards with red packaging", + "rectangular red playingcards holder" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/081_playingcards/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/081_playingcards/base2.json new file mode 100644 index 00000000..b216e1f9 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/081_playingcards/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "playingcards", + "seen": [ + "box with cards inside", + "black playing cards box", + "box holding playing cards", + "rectangle-shaped cards box", + "small black box with cards", + "cardboard playing cards box", + "smooth black cards container", + "compact box for playing cards", + "palm-sized box for playing cards", + "playing cards box with gold text", + "black box with rectangular shape", + "rectangular box for card storage" + ], + "unseen": [ + "hard black box for card deck", + "black box with smooth surface", + "gold text on playing cards box" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/082_smallshovel/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/082_smallshovel/base0.json new file mode 100644 index 00000000..6eb76b4a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/082_smallshovel/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "smallshovel", + "seen": [ + "compact smallshovel", + "handheld smallshovel", + "light gray smallshovel", + "metal handle smallshovel", + "smallshovel with rounded scoop", + "smallshovel with plastic scoop", + "rounded scoop gray smallshovel", + "smallshovel with smooth surface", + "smallshovel with curved edge scoop", + "smallshovel for small digging tasks", + "light gray smallshovel with black tip", + "smallshovel with short straight handle" + ], + "unseen": [ + "rubber grip smallshovel", + "light gray scoop smallshovel", + "smallshovel with black grip handle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/082_smallshovel/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/082_smallshovel/base1.json new file mode 100644 index 00000000..903ce45e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/082_smallshovel/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "smallshovel", + "seen": [ + "black smallshovel", + "black metal smallshovel", + "metal blade smallshovel", + "wooden-handled smallshovel", + "smallshovel for digging dirt", + "flat-edged black smallshovel", + "light wooden smallshovel handle", + "smallshovel with hole at handle", + "black smallshovel with flat edges", + "smallshovel with rectangular blade", + "smallshovel with sturdy wooden grip", + "short smallshovel with smooth handle" + ], + "unseen": [ + "hand-sized black smallshovel", + "smallshovel with light wood handle", + "compact smallshovel with strong blade" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/082_smallshovel/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/082_smallshovel/base2.json new file mode 100644 index 00000000..10f0ee59 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/082_smallshovel/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "smallshovel", + "seen": [ + "smallshovel for scooping", + "reddish brown smallshovel", + "smooth reddish plastic scoop", + "rounded red tool smallshovel", + "smallshovel with curved edges", + "hand-sized plastic smallshovel", + "compact lightweight smallshovel", + "curved base reddish smallshovel", + "reddish brown scoop smallshovel", + "short-handled plastic smallshovel", + "red smallshovel with smooth finish", + "smallshovel made of smooth plastic" + ], + "unseen": [ + "rounded red scoop", + "rounded edge smallshovel", + "smallshovel with short handle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/082_smallshovel/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/082_smallshovel/base3.json new file mode 100644 index 00000000..f256d872 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/082_smallshovel/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "smallshovel", + "seen": [ + "small yellow scoop", + "half-circle smallshovel", + "light yellow hand-scoop", + "light yellow plastic shovel", + "smallshovel with flat handle", + "hand-held yellow smallshovel", + "curved edge yellow smallshovel", + "yellow smallshovel for scooping", + "smooth yellow half-circle scoop", + "short-handled light yellow shovel", + "yellow curved plastic smallshovel", + "smallshovel with smooth plastic surface" + ], + "unseen": [ + "hand-sized scoop", + "yellow smallshovel", + "plastic scoop for small tasks" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/083_brush/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/083_brush/base0.json new file mode 100644 index 00000000..27e52cc3 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/083_brush/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "brush", + "seen": [ + "dark bristle brush", + "flat oval-shaped brush", + "blue-handled hair brush", + "medium-sized hair brush", + "oval black bristle brush", + "blue-handled oval hair brush", + "brush with textured black head", + "brush with rough black bristles", + "black oval brush with white back", + "oval black-head brush with blue grip", + "plastic brush with synthetic bristles", + "dark bristle brush with smooth handle" + ], + "unseen": [ + "brush with blue handle", + "medium flat brush for hair", + "brush with smooth white backing" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/083_brush/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/083_brush/base1.json new file mode 100644 index 00000000..b517e3f5 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/083_brush/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "brush", + "seen": [ + "white brush", + "cleaning brush", + "brush with gray handle", + "handheld cleaning brush", + "smooth gray-handle brush", + "medium-sized plastic brush", + "brush for scrubbing surfaces", + "synthetic brush for cleaning", + "black-bristled cleaning brush", + "white and black cleaning brush", + "brush with long tapered handle", + "white rectangular brush with black bristles" + ], + "unseen": [ + "plastic brush with stiff bristles", + "brush with rectangular bristle head", + "gray-handled brush with black bristles" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/083_brush/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/083_brush/base2.json new file mode 100644 index 00000000..85c930b4 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/083_brush/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "brush", + "seen": [ + "gray body brush", + "black bristle brush", + "simple handheld brush", + "medium-sized cleaning brush", + "flat gray brush for sweeping", + "gray brush with black bristles", + "plastic brush with long handle", + "brush with coarse bristle head", + "cleaning brush with dark handle", + "flat brush with rectangular head", + "black and gray rectangular brush", + "rectangular brush with thin bristles" + ], + "unseen": [ + "sweeping brush with black bristles", + "gray plastic brush with black ends", + "brush with lightweight plastic design" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/083_brush/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/083_brush/base3.json new file mode 100644 index 00000000..fb5fff86 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/083_brush/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "brush", + "seen": [ + "yellow brush", + "plastic cleaning brush", + "plastic bristles brush", + "long-handle yellow brush", + "light yellow handle brush", + "yellow brush with bristles", + "yellow brush for scrubbing", + "compact rectangular brush head", + "medium yellow brush with handle", + "cleaning brush with hole in handle", + "yellow brush for cleaning surfaces", + "light yellow plastic scrubbing brush" + ], + "unseen": [ + "smooth handle with bristle head", + "brush with rigid yellow bristles", + "brush with rectangular bristle head" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/084_woodenmallet/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/084_woodenmallet/base0.json new file mode 100644 index 00000000..01f9cfa3 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/084_woodenmallet/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "woodenmallet", + "seen": [ + "wooden mallet", + "light brown mallet", + "mallet with wooden finish", + "wooden mallet for striking", + "rounded-head wooden mallet", + "mallet with cylindrical head", + "hammer-shaped light brown mallet", + "mallet with sturdy wooden design", + "wooden mallet with smooth surface", + "mallet with tapered wooden handle", + "light brown polished wooden mallet", + "light brown hammer-shaped wooden mallet" + ], + "unseen": [ + "handheld wooden mallet", + "mallet with smooth texture", + "medium-sized wooden mallet" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/084_woodenmallet/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/084_woodenmallet/base1.json new file mode 100644 index 00000000..b60945a4 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/084_woodenmallet/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "wooden mallet", + "seen": [ + "wooden mallet", + "light brown mallet", + "simple wooden mallet", + "smooth light wood mallet", + "mallet with smooth surface", + "medium-sized wooden mallet", + "mallet with round wooden head", + "light wooden tool with handle", + "mallet with rounded hammer ends", + "hammer with wooden head and handle", + "wood hammer with cylindrical handle", + "mallet with polished light brown finish" + ], + "unseen": [ + "mallet made of wood", + "wooden mallet for striking", + "round hammerhead wooden mallet" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/084_woodenmallet/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/084_woodenmallet/base2.json new file mode 100644 index 00000000..af7ab6d7 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/084_woodenmallet/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "woodenmallet", + "seen": [ + "wooden mallet", + "round mallet head", + "light brown mallet", + "smooth wooden mallet", + "mallet for hammering", + "wooden mallet handle and head", + "light brown round hammer tool", + "light brown rounded hammer tool", + "medium-sized light brown mallet", + "rounded wooden hammer-like mallet", + "smooth mallet handle and flat head", + "mallet for tapping with wood surface" + ], + "unseen": [ + "mallet with long handle", + "mallet with flat round head", + "rounded mallet head with flat ends" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/084_woodenmallet/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/084_woodenmallet/base3.json new file mode 100644 index 00000000..dd13f20a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/084_woodenmallet/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "woodenmallet", + "seen": [ + "brown mallet", + "solid wooden mallet", + "wooden mallet for crafts", + "mallet with block and grip", + "mallet with rectangular head", + "tool with long wooden handle", + "mallet with smooth light wood", + "light brown mallet with handle", + "medium mallet for shaping wood", + "mallet with rectangular block head", + "blocky mallet with tall grip handle", + "woodworking mallet with soft finish" + ], + "unseen": [ + "wooden mallet", + "smooth wooden tool for striking", + "hammer-style mallet made of wood" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/084_woodenmallet/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/084_woodenmallet/base4.json new file mode 100644 index 00000000..9bfc5895 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/084_woodenmallet/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "woodenmallet", + "seen": [ + "wooden mallet", + "smooth wooden mallet", + "light brown wooden mallet", + "mallet with wooden handle", + "wooden hammer-like mallet", + "mallet used for hammering tasks", + "medium mallet made of polished wood", + "mallet for striking with wooden head", + "cylindrical head mallet made of wood", + "wooden mallet with long curved handle", + "brown wooden mallet with smooth finish", + "lightweight wooden mallet with rounded handle" + ], + "unseen": [ + "hammer-like wooden mallet", + "mallet with smooth texture", + "light wood mallet with cylinder head" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/085_gong/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/085_gong/base0.json new file mode 100644 index 00000000..56a51346 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/085_gong/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "gong", + "seen": [ + "golden brown gong", + "round metallic gong", + "gong with handles on sides", + "metal gong with smooth finish", + "flat gong with circular ridges", + "gong held by brown wooden frame", + "flat disc gong with raised center", + "gong for producing resonant sounds", + "gong attached to rectangular stand", + "medium-sized gong with wooden frame", + "polished golden gong with curved handles", + "gong surrounded by smooth wooden support" + ], + "unseen": [ + "golden gong with raised middle", + "gong mounted on dark brown base", + "gong hanging on rectangular wooden frame" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/085_gong/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/085_gong/base1.json new file mode 100644 index 00000000..72847b53 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/085_gong/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "gong", + "seen": [ + "golden gong", + "wood-framed gong", + "gong for striking", + "round metallic gong", + "gong on wooden frame", + "smooth metallic gong", + "gong with wooden stand", + "gong with raised center", + "gong with circular design", + "metal gong with wooden holder", + "golden gong with polished wood", + "round gong with smooth surface" + ], + "unseen": [ + "tabletop gong", + "golden brown gong", + "medium-sized gong" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/085_gong/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/085_gong/base2.json new file mode 100644 index 00000000..905968da --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/085_gong/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "gong", + "seen": [ + "round bronze musical gong", + "metal gong on wooden stand", + "wooden frame with round gong", + "medium gong mounted on frame", + "gong and sturdy wooden frame", + "gong disc with smooth texture", + "bronze gong with curved center", + "large gong with wooden support", + "circular gong with carved edges", + "traditional gong with smooth surface", + "metallic gong resting on wooden legs", + "gong mounted securely on wooden stand" + ], + "unseen": [ + "bronze gong", + "striking gong with wood base", + "gong with deep metallic shine" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/085_gong/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/085_gong/base3.json new file mode 100644 index 00000000..2323ba11 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/085_gong/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "gong", + "seen": [ + "circular gong", + "smooth disk gong", + "golden brown gong", + "gold-bronze gong plate", + "medium-sized round gong", + "gong with central round knob", + "golden gong on a wooden frame", + "flat circular gong with ridges", + "mounted gong with smooth texture", + "metal gong mounted on square frame", + "gong with centered striking surface", + "brown metallic gong with knob center" + ], + "unseen": [ + "wood-framed golden gong", + "medium gong with a square base", + "disk-shaped musical gong on frame" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/085_gong/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/085_gong/base4.json new file mode 100644 index 00000000..1d521415 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/085_gong/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "gong", + "seen": [ + "brown gong", + "wooden gong", + "light wooden gong", + "small tabletop gong", + "light brown gong setup", + "gong with smooth finish", + "gong with rounded edges", + "gong with circular shape", + "medium-sized gong holder", + "wood gong mounted on frame", + "gong with polished surface", + "circular gong on wooden stand" + ], + "unseen": [ + "gong with dome center", + "flat gong with raised dome", + "gong frame with intricate design" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/085_gong/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/085_gong/base5.json new file mode 100644 index 00000000..05940f68 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/085_gong/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "gong", + "seen": [ + "round gong", + "metal gong", + "golden gong", + "flat circular gong", + "bronze disc-shaped gong", + "medium gold-colored gong", + "golden ring-patterned gong", + "gong with concentric ridges", + "golden flat percussion gong", + "smooth metallic gong with stand", + "bronze gong for striking sounds", + "plate-sized gong with central bump" + ], + "unseen": [ + "bronze gong", + "gong with stand", + "gong on upright frame" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/086_woodenblock/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/086_woodenblock/base0.json new file mode 100644 index 00000000..0fe7aca4 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/086_woodenblock/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "woodenblock", + "seen": [ + "brown woodenblock", + "smooth woodenblock", + "shaped woodenblock", + "block made from wood", + "hand-sized woodenblock", + "decorative woodenblock", + "brown smooth woodenblock", + "hand-sized block of wood", + "woodenblock with brown finish", + "brown block with wooden texture", + "woodenblock with smooth surface", + "woodenblock for structural purposes" + ], + "unseen": [ + "medium brown block", + "irregular woodenblock", + "medium irregular woodenblock" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/086_woodenblock/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/086_woodenblock/base1.json new file mode 100644 index 00000000..39e566d4 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/086_woodenblock/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "woodenblock", + "seen": [ + "small woodenblock", + "solid woodenblock", + "smooth woodenblock", + "palm-sized woodenblock", + "light brown woodenblock", + "circular-ended woodenblock", + "cylinder-shaped woodenblock", + "grainy-textured woodenblock", + "small light brown woodenblock", + "woodenblock with rounded edges", + "smooth light brown woodenblock", + "light brown cylinder woodenblock" + ], + "unseen": [ + "woodenblock for stacking", + "woodenblock with circular sides", + "rounded woodenblock with grain texture" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/086_woodenblock/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/086_woodenblock/base2.json new file mode 100644 index 00000000..a5c81044 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/086_woodenblock/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "woodenblock", + "seen": [ + "orange and green woodenblock", + "medium-sized woodenblock for stacking", + "woodenblock with triangular green side", + "smooth woodenblock with colorful design", + "building block with green triangle shape", + "woodenblock with triangular green attachment", + "vivid orange block and bright green triangle", + "block for building with triangular green edge", + "colorful woodenblock with geometric green side", + "orange woodenblock with smooth green triangular part", + "orange rectangular woodenblock with added green triangle", + "rectangular woodenblock base and attached green triangle" + ], + "unseen": [ + "medium woodenblock with smooth texture", + "rectangular orange block with green triangle", + "orange block paired with green triangular piece" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/086_woodenblock/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/086_woodenblock/base3.json new file mode 100644 index 00000000..5d6d9f5f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/086_woodenblock/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "woodenblock", + "seen": [ + "medium woodenblock", + "light brown woodenblock", + "rectangular woodenblock", + "rectangular block made of wood", + "light brown rectangular wood block", + "building block with smooth surface", + "medium rectangle-shaped woodenblock", + "light brown woodenblock for building", + "woodenblock medium size smooth texture", + "rectangular woodenblock with smooth grain", + "woodenblock light color rectangular prism", + "rectangular wood block with grain texture" + ], + "unseen": [ + "smooth woodenblock", + "smooth woodenblock made of light wood", + "medium rectangular woodenblock with grain" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/086_woodenblock/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/086_woodenblock/base4.json new file mode 100644 index 00000000..c4ce1526 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/086_woodenblock/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "woodenblock", + "seen": [ + "woodenblock with front slot", + "woodenblock with back grooves", + "simple woodenblock with slots", + "woodenblock with rounded edges", + "smooth rectangular woodenblock", + "woodenblock with recessed area", + "woodenblock with vertical slits", + "compact woodenblock for organizing", + "light brown woodenblock with patterns", + "polished woodenblock with grain texture", + "woodenblock with slotted design on side", + "solid woodenblock with multi-sided features" + ], + "unseen": [ + "hand-sized woodenblock", + "light brown woodenblock", + "rectangular woodenblock" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/087_waterer/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/087_waterer/base0.json new file mode 100644 index 00000000..b8fb8381 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/087_waterer/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "waterer", + "seen": [ + "plastic watering can", + "watering can for plants", + "light green plant waterer", + "smooth green watering tool", + "light green garden waterer", + "medium garden watering can", + "medium watering can with spout", + "green cylindrical watering can", + "light green water pouring tool", + "watering can with curved handle", + "plastic can for watering plants", + "smooth watering can with narrow spout" + ], + "unseen": [ + "green watering can", + "light green watering container", + "cylinder can with handle and spout" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/087_waterer/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/087_waterer/base1.json new file mode 100644 index 00000000..ff59af40 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/087_waterer/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "waterer", + "seen": [ + "yellow watering can", + "handheld watering can", + "medium-sized watering can", + "bright yellow garden waterer", + "yellow plastic plant waterer", + "watering can with long spout", + "smooth plastic yellow waterer", + "yellow can for garden watering", + "yellow watering can with smooth body", + "bright yellow can for watering plants", + "medium watering can for pouring water", + "watering can with cylindrical tank and spout" + ], + "unseen": [ + "waterer with handle and spout", + "plastic watering can with curved handle", + "yellow plant waterer with cylindrical body" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/087_waterer/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/087_waterer/base2.json new file mode 100644 index 00000000..3b807a9e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/087_waterer/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "waterer", + "seen": [ + "plastic waterer", + "blue watering can", + "dark blue plant waterer", + "cylindrical watering can", + "medium-sized watering can", + "watering can with long spout", + "handheld plastic watering can", + "blue body with attached spout", + "plant waterer with curved handle", + "watering tool with curved handle", + "waterer with smooth plastic finish", + "blue cylinder with spout and handle" + ], + "unseen": [ + "smooth watering can", + "watering can for plants", + "dark blue watering container" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/087_waterer/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/087_waterer/base3.json new file mode 100644 index 00000000..a0b07edc --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/087_waterer/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "waterer", + "seen": [ + "green waterer", + "smooth watering can", + "medium-sized waterer", + "bright green watering can", + "plastic green waterer body", + "medium waterer with yellow spout", + "plastic waterer with curved handle", + "bright green and yellow watering tool", + "small watering can with smooth texture", + "plant watering tool with handle and spout", + "rounded green water container with handle", + "green waterer with perforated yellow spout" + ], + "unseen": [ + "green rounded water holder", + "yellow-spouted green plastic waterer", + "green waterer with semicircular handle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/087_waterer/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/087_waterer/base4.json new file mode 100644 index 00000000..2d63e021 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/087_waterer/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "waterer", + "seen": [ + "green waterer", + "green water can", + "light green watering can", + "plastic green watering can", + "handheld green watering can", + "plant watering green waterer", + "green waterer with long spout", + "green waterer for plant watering", + "plastic green waterer with spout", + "green waterer with attached handle", + "green watering can with rounded body", + "medium green waterer with flat nozzle" + ], + "unseen": [ + "smooth green waterer", + "cylindrical green waterer", + "medium green waterer with handle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/087_waterer/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/087_waterer/base5.json new file mode 100644 index 00000000..eb4a5ca5 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/087_waterer/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "waterer", + "seen": [ + "green watering can", + "handheld green watering can", + "watering can with long spout", + "green can with curved handle", + "bright green plant watering tool", + "bright green plastic watering can", + "watering can with spout and handle", + "green plastic can with pouring spout", + "plastic green cylindrical watering can", + "garden watering can with smooth surface", + "watering can with loop handle and nozzle", + "medium-sized green water container for plants" + ], + "unseen": [ + "smooth green watering can", + "medium green can for watering plants", + "green cylindrical body with long spout" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/087_waterer/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/087_waterer/base6.json new file mode 100644 index 00000000..32d2ec57 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/087_waterer/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "waterer", + "seen": [ + "green waterer", + "plastic waterer", + "medium waterer for plants", + "plant watering can in green", + "plastic green watering tool", + "medium green garden waterer", + "smooth green plastic waterer", + "green waterer with long spout", + "round body waterer with spout", + "watering can with curved handle", + "medium watering can for gardens", + "green bucket-shaped waterer with handle" + ], + "unseen": [ + "medium green watering tool", + "green waterer with tapered nozzle", + "green waterer with cylindrical base" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/087_waterer/base7.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/087_waterer/base7.json new file mode 100644 index 00000000..5f82c568 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/087_waterer/base7.json @@ -0,0 +1,22 @@ +{ + "raw_description": "waterer", + "seen": [ + "blue watering can", + "plant watering can", + "smooth blue watering can", + "medium blue plastic waterer", + "medium plastic watering can", + "blue can with white spout cap", + "plastic can with smooth finish", + "plant waterer with curved spout", + "blue plant waterer with green details", + "rounded watering can with curved spout", + "cylindrical waterer with handle and spout", + "blue can with painted green and beige patterns" + ], + "unseen": [ + "blue can for gardening", + "decorated blue water can", + "blue gardener's watering can" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/088_wineglass/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/088_wineglass/base0.json new file mode 100644 index 00000000..d45cfd39 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/088_wineglass/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "wineglass", + "seen": [ + "medium wineglass", + "smooth wineglass", + "shiny stem wineglass", + "transparent wineglass", + "rounded wineglass cup", + "wineglass with black stem", + "wineglass with tall thin stem", + "wineglass cup with black base", + "wineglass with wide round bowl", + "wineglass used for holding wine", + "wineglass bowl with curved edges", + "wineglass with glossy glass bowl" + ], + "unseen": [ + "black base wineglass", + "black and transparent wineglass", + "wineglass with sturdy black base" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/088_wineglass/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/088_wineglass/base1.json new file mode 100644 index 00000000..835152bb --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/088_wineglass/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "wineglass", + "seen": [ + "clear wineglass", + "glass wineglass", + "classic wineglass", + "wineglass with round base", + "medium wineglass for wine", + "smooth transparent wineglass", + "round wineglass with slim stem", + "glass wineglass with bowl shape", + "wineglass with shiny glass look", + "wineglass with small round foot", + "curved wineglass with stem handle", + "stemmed wineglass with clear glass" + ], + "unseen": [ + "tall wineglass", + "slim wineglass with glossy surface", + "transparent wineglass with long stem" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/088_wineglass/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/088_wineglass/base2.json new file mode 100644 index 00000000..affee236 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/088_wineglass/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "wineglass", + "seen": [ + "tall wineglass", + "stemmed wineglass", + "smooth glass wineglass", + "wineglass with thin stem", + "elongated glass wineglass", + "wineglass with round bowl", + "transparent wineglass bowl", + "wineglass for serving wine", + "wineglass with stable base", + "wineglass made of clear glass", + "wineglass with polished texture", + "wineglass for red or white wine" + ], + "unseen": [ + "clear wineglass", + "cylindrical wineglass bowl", + "wineglass with wide flat base" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/088_wineglass/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/088_wineglass/base3.json new file mode 100644 index 00000000..a2396835 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/088_wineglass/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "wineglass", + "seen": [ + "clear wineglass", + "glass wineglass", + "smooth wineglass", + "handheld wineglass", + "wineglass with dark stem", + "wineglass with thin stem", + "glass wineglass for beverages", + "clear wineglass with wide bowl", + "wineglass with flared top edge", + "glass wineglass with round base", + "curved wineglass with small base", + "stemmed wineglass with curved top" + ], + "unseen": [ + "wineglass for drinking wine", + "wineglass with polished surface", + "medium wineglass with curved bowl" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/088_wineglass/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/088_wineglass/base4.json new file mode 100644 index 00000000..f60ca162 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/088_wineglass/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "wineglass", + "seen": [ + "clear wineglass", + "dark-based wineglass", + "medium-sized wineglass", + "rounded glass for wine", + "stemmed glass for wine", + "glass with rounded bowl", + "wineglass with dark base", + "clear glass bowl with base", + "white glass with dark stand", + "wineglass with shiny surface", + "wineglass for holding liquids", + "clear wineglass with smooth texture" + ], + "unseen": [ + "wineglass", + "wineglass with slim stem", + "glossy wineglass with rounded top" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/089_globe/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/089_globe/base2.json new file mode 100644 index 00000000..50055602 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/089_globe/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "globe", + "seen": [ + "Earth map globe", + "round Earth globe", + "decor piece globe", + "spinning brown globe", + "beige and brown globe", + "globe with smooth surface", + "brown globe on black stand", + "wooden globe with metal base", + "round globe with sturdy base", + "globe with curved black stand", + "globe for geographic reference", + "smooth beige globe with continents" + ], + "unseen": [ + "brown globe", + "tabletop globe", + "medium-sized rotating globe" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/089_globe/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/089_globe/base3.json new file mode 100644 index 00000000..35f41c70 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/089_globe/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "globe", + "seen": [ + "blue globe", + "small world globe", + "globe with curved black base", + "colorful globe on black stand", + "small sphere showing earth map", + "hand-sized globe on sturdy stand", + "blue sphere with countries marked", + "globe sitting on curved black base", + "blue globe with colorful continents", + "earth-shaped globe with black frame", + "compact globe with labeled countries", + "globe featuring bright country colors" + ], + "unseen": [ + "smooth blue globe with stand", + "globe with clear ocean markings", + "ball-shaped globe with metal holder" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/090_trophy/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/090_trophy/base0.json new file mode 100644 index 00000000..62362f59 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/090_trophy/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "trophy", + "seen": [ + "trophy cup", + "gold trophy", + "award trophy", + "gold prize cup", + "cup-shaped trophy", + "trophy with curved handles", + "shiny gold cup with black stand", + "medium trophy with pedestal base", + "trophy with smooth golden surface", + "decorative trophy with shiny finish", + "smooth gold cup with sturdy pedestal", + "gold-colored prize cup with solid base" + ], + "unseen": [ + "golden cup with black base", + "gold cup with side handles", + "golden award with rectangular black base" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/090_trophy/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/090_trophy/base1.json new file mode 100644 index 00000000..51ee23be --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/090_trophy/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "trophy", + "seen": [ + "golden trophy", + "shiny yellow trophy", + "award trophy with two handles", + "golden cup with curved handles", + "trophy with sleek golden finish", + "metallic yellow cup-shaped trophy", + "tall yellow trophy with small neck", + "smooth metallic cup with flat bottom", + "shiny trophy for winning competitions", + "tall award trophy with glossy surface", + "golden yellow cup-shaped prize trophy", + "achievement cup with curved side handles" + ], + "unseen": [ + "smooth trophy with wide base", + "medium-sized golden prize cup", + "prize trophy with sturdy round base" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/090_trophy/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/090_trophy/base2.json new file mode 100644 index 00000000..f4a9b8a2 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/090_trophy/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "trophy", + "seen": [ + "trophy cup", + "award trophy", + "golden trophy", + "gold cup with handles", + "two-handled prize trophy", + "shiny golden prize trophy", + "medium-sized yellow trophy", + "smooth gold cup-shaped trophy", + "golden award with orange edges", + "glossy yellow trophy with base", + "award cup with orange highlights", + "decorative trophy with curved handles" + ], + "unseen": [ + "yellow cup with round stand", + "hand-sized shiny gold trophy", + "golden trophy with round base" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/090_trophy/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/090_trophy/base3.json new file mode 100644 index 00000000..c1dfe019 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/090_trophy/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "trophy", + "seen": [ + "gold trophy", + "ornate gold cup", + "gold cup on black base", + "gold and black prize cup", + "medium-sized prize trophy", + "two-handled golden trophy", + "trophy with two side handles", + "award trophy with black platform", + "trophy with detailed top and base", + "shiny gold trophy with black stand", + "trophy with smooth polished finish", + "centerpiece gold trophy on sturdy base" + ], + "unseen": [ + "smooth gold cup", + "golden award cup", + "trophy cup with curved handles" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/090_trophy/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/090_trophy/base4.json new file mode 100644 index 00000000..6f07da1b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/090_trophy/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "trophy", + "seen": [ + "gold trophy", + "medium-sized golden trophy", + "shiny trophy with top knob", + "trophy with dangling ribbons", + "trophy cup with curved sides", + "cup-shaped trophy with handles", + "gold award cup with two handles", + "gold trophy on square black base", + "golden award trophy with black base", + "decorative trophy with curved handles", + "black bottom trophy with smooth finish", + "medium trophy with gold and black colors" + ], + "unseen": [ + "shiny trophy cup", + "smooth golden trophy", + "trophy with red and blue ribbons" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/091_kettle/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/091_kettle/base0.json new file mode 100644 index 00000000..af3cf0be --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/091_kettle/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "kettle", + "seen": [ + "silver kettle", + "rounded kettle with spout", + "smooth-surface silver kettle", + "medium-sized countertop kettle", + "metal kettle for boiling water", + "kettle with matte black handle", + "kettle with curved pouring spout", + "modern kettle with sturdy handle", + "compact kettle for heating water", + "rounded silver kettle for kitchen use", + "silver metal kettle with plastic grip", + "teapot-shaped kettle with smooth finish" + ], + "unseen": [ + "kettle with black handle", + "boiling kettle with wide base", + "shiny silver kettle with black grip" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/091_kettle/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/091_kettle/base1.json new file mode 100644 index 00000000..7fea7b53 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/091_kettle/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "kettle", + "seen": [ + "green kettle", + "golden spout kettle", + "dark green stove kettle", + "dark green kettle with lid", + "medium glossy kettle with handle", + "rounded kettle with curved spout", + "metal kettle with plastic handle", + "medium kettle with glossy finish", + "stovetop kettle with black handle", + "curved spout kettle with black grip", + "water-boiling green and golden kettle", + "rounded body kettle with golden accents" + ], + "unseen": [ + "green kettle with golden lid", + "medium-sized green water kettle", + "smooth-textured kettle with metallic parts" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/091_kettle/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/091_kettle/base2.json new file mode 100644 index 00000000..22da6b7a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/091_kettle/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "kettle", + "seen": [ + "silver kettle", + "rounded shiny kettle", + "stainless steel kettle", + "kettle for boiling water", + "kettle with sturdy build", + "metal kettle with black handle", + "metal kettle with curved spout", + "kettle with curved pouring spout", + "silver kettle with smooth finish", + "stovetop kettle with silver body", + "silver kettle with polished surface", + "shiny kettle with black plastic handle" + ], + "unseen": [ + "medium-sized stovetop kettle", + "classic kettle design with spout", + "black handle and spout on kettle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/091_kettle/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/091_kettle/base3.json new file mode 100644 index 00000000..395b9015 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/091_kettle/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "kettle", + "seen": [ + "red kettle", + "metal kettle", + "kettle with spout", + "small bright red kettle", + "medium red water kettle", + "smooth glossy red kettle", + "red kettle with black handle", + "red kettle with curved spout", + "red kettle with glossy finish", + "red kettle with smooth texture", + "metal kettle with shiny coating", + "round red kettle for boiling water" + ], + "unseen": [ + "red round kettle", + "medium red kettle with handle", + "bright red kettle with dark handle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/091_kettle/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/091_kettle/base4.json new file mode 100644 index 00000000..8d8eca46 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/091_kettle/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "kettle", + "seen": [ + "blue and white kettle", + "glossy ceramic kettle", + "kettle for boiling water", + "kettle with spout and lid", + "lid with blue knob on kettle", + "tea kettle with floral design", + "ceramic kettle with floral prints", + "kitchen kettle with wooden handle", + "bulbous kettle with pointed spout", + "smooth ceramic kettle with prints", + "medium-sized white kettle with blue", + "decorative kettle with brown handle" + ], + "unseen": [ + "brown wooden handle kettle", + "white kettle with blue flowers", + "rounded kettle with curved handle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/091_kettle/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/091_kettle/base5.json new file mode 100644 index 00000000..6ac1e216 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/091_kettle/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "kettle", + "seen": [ + "round orange kettle", + "medium-sized stovetop kettle", + "kettle with black grip handle", + "metal kettle with black handle", + "orange kettle with small spout", + "metal kettle with curved spout", + "rounded kettle with shiny finish", + "light orange kettle with smooth lid", + "medium stovetop kettle with flat lid", + "light orange kettle with curved spout", + "smooth orange kettle with ribbed handle", + "stovetop kettle with ribbed black handle" + ], + "unseen": [ + "metal kettle with round lid", + "orange kettle for boiling water", + "orange kettle with angled spout" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/092_notebook/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/092_notebook/base0.json new file mode 100644 index 00000000..7412cf03 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/092_notebook/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "notebook", + "seen": [ + "blue notebook", + "blue notebook for notes", + "medium dark blue notebook", + "dark blue hardcover notebook", + "notebook with smooth hardcover", + "writing notebook with dark cover", + "dark blue notebook with hardcover", + "medium notebook shaped like a book", + "hand-sized notebook with silver edge", + "notebook with glossy dark blue finish", + "note-taking notebook with shiny cover", + "rectangular notebook with tapering spine" + ], + "unseen": [ + "glossy silver-edged notebook", + "shiny notebook with slim spine", + "medium notebook with shiny cover" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/092_notebook/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/092_notebook/base1.json new file mode 100644 index 00000000..d0018ab0 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/092_notebook/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "notebook", + "seen": [ + "light gray notebook", + "rectangular notebook", + "medium-sized notebook", + "gray notebook for notes", + "notebook with paper pages", + "notebook with smooth cover", + "simple light gray notebook", + "notebook with smooth texture", + "gray rectangular writing notebook", + "writing notebook with medium size", + "medium notebook with paper inside", + "notebook with smooth light gray look" + ], + "unseen": [ + "notebook for writing", + "notebook with cardboard cover", + "notebook for sketches and notes" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/092_notebook/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/092_notebook/base2.json new file mode 100644 index 00000000..fedd1381 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/092_notebook/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "notebook", + "seen": [ + "black notebook", + "matte black notebook", + "flat rectangular notebook", + "hand-sized black notebook", + "small rectangular notebook", + "hard-covered black notebook", + "notebook with white page edges", + "black notebook for note-taking", + "writing notebook with hard cover", + "compact notebook with black cover", + "rectangular notebook with smooth texture", + "simple rectangular notebook with black cover" + ], + "unseen": [ + "paper notebook with black exterior", + "black notebook with white paper edges", + "small black notebook with smooth pages" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/093_brush-pen/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/093_brush-pen/base0.json new file mode 100644 index 00000000..b7d059a2 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/093_brush-pen/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "brush-pen", + "seen": [ + "slim brush-pen", + "glossy brush-pen", + "plastic brush-pen", + "drawing brush-pen", + "pen for artistic drawing", + "smooth black and beige pen", + "pen with pointed white tip", + "beige pen with black accents", + "slim pen with brush-like tip", + "beige body pen with black ends", + "brush-pen with tapered white tip", + "white-tipped brush-pen for lines" + ], + "unseen": [ + "hand-sized brush-pen", + "black and beige brush-pen", + "cylindrical beige brush-pen" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/093_brush-pen/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/093_brush-pen/base1.json new file mode 100644 index 00000000..a589448c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/093_brush-pen/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "brush pen", + "seen": [ + "brush pen", + "hand-sized brush pen", + "pointed tip brush pen", + "pen with orange bristle tip", + "long brush pen with soft tip", + "smooth brush pen for painting", + "thin black pen with brown end", + "orange-tipped pen black handle", + "black and brown pen orange end", + "orange soft tip black brown pen", + "slender black pen with orange point", + "painting pen with synthetic bristles" + ], + "unseen": [ + "plastic barrel brush pen", + "drawing pen with soft orange tip", + "brown base black barrel orange tip" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/093_brush-pen/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/093_brush-pen/base2.json new file mode 100644 index 00000000..ed4e6030 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/093_brush-pen/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "brush-pen", + "seen": [ + "brown brush-pen", + "long brown brush-pen", + "pen-shaped brush-pen", + "brush-pen for drawing", + "pointed-tip brush-pen", + "brown barrel brush-pen", + "brush-pen with soft tip", + "smooth plastic brush-pen", + "brown and yellow brush-pen", + "synthetic fiber tip brush-pen", + "yellow-green patterned brush-pen", + "handheld brush-pen with patterns" + ], + "unseen": [ + "thin writing brush-pen", + "slim pen with brush end", + "brush-pen for artistic work" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/093_brush-pen/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/093_brush-pen/base3.json new file mode 100644 index 00000000..31cfa098 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/093_brush-pen/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "brush-pen", + "seen": [ + "black and beige brush-pen", + "writing or painting brush-pen", + "painting brush with black tip", + "small black tip on beige shaft", + "long beige handle with black tip", + "round wooden handle of brush-pen", + "smooth wooden handle of brush-pen", + "handheld brush-pen for art or writing", + "beige handle with soft black bristles", + "soft black brush attached to beige handle", + "beige cylindrical handle with black brush", + "synthetic black bristles on tapered handle" + ], + "unseen": [ + "black pointed brush tip", + "precision writing brush-pen", + "palm-sized artistic brush-pen" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/093_brush-pen/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/093_brush-pen/base4.json new file mode 100644 index 00000000..c2bbdf8b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/093_brush-pen/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "brush-pen", + "seen": [ + "dark gray brush-pen", + "triangular brush-pen", + "compact handheld brush-pen", + "long dark gray art brush-pen", + "dark gray smooth brush-pen for art", + "drawing brush-pen with triangular body", + "medium-sized triangular gray brush-pen", + "triangular gray brush-pen for sketches", + "plastic handheld brush-pen in dark gray", + "lightweight brush-pen with smooth finish", + "writing brush-pen with sleek triangular shape", + "dark gray triangular pen for artistic strokes" + ], + "unseen": [ + "plastic brush-pen for drawing", + "smooth gray triangular brush-pen", + "dark gray triangular writing tool" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/093_brush-pen/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/093_brush-pen/base5.json new file mode 100644 index 00000000..a7bd42e6 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/093_brush-pen/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "brush pen", + "seen": [ + "brown brush pen", + "brown body brush pen", + "thin pen for drawing", + "brush pen used for art", + "brush pen with black grip", + "brush pen with silver tip", + "plastic pen with brush end", + "slim brown painting brush pen", + "brown pen with silver ferrule", + "long brush pen with tapered end", + "brown drawing pen with tapered tip", + "brown cylindrical pen for painting" + ], + "unseen": [ + "smooth pen with brush end", + "brush pen with narrow body", + "long brown pen with black handle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/094_rest/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/094_rest/base0.json new file mode 100644 index 00000000..2cb49077 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/094_rest/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "rest", + "seen": [ + "brown rest", + "light brown ridged rest", + "smooth rectangular rest", + "medium-size plastic rest", + "rest with ridge patterns", + "palm-length rectangular rest", + "plastic rest with raised ridges", + "smooth brown rectangular holder", + "light brown ridged plastic holder", + "rectangular rest with raised edges", + "medium-size smooth light-brown base", + "brown ridged smooth rectangular rest" + ], + "unseen": [ + "rest for holding or supporting", + "light brown smooth ridged block", + "brown base with ridge-like shapes" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/094_rest/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/094_rest/base1.json new file mode 100644 index 00000000..bbe54f9d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/094_rest/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "rest", + "seen": [ + "wavy green block", + "abstract green rest block", + "glass-like green rest block", + "translucent green wavy block", + "transparent green rest model", + "medium-sized green rest block", + "medium rest with glassy texture", + "green block held within box frame", + "smooth green block with wavy edges", + "rest inside clear rectangular frame", + "green wavy block inside a clear box", + "green block with uneven wave-like shape" + ], + "unseen": [ + "green rest", + "irregular green block inside box", + "green block encased in rectangular prism" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/094_rest/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/094_rest/base2.json new file mode 100644 index 00000000..c21ebbe2 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/094_rest/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "rest", + "seen": [ + "green triangular rest", + "green wedge with red edge", + "hand-sized green crackled rest", + "green crackled triangular wedge", + "triangular rest with jagged side", + "ceramic green rest with red trim", + "textured green wedge-shaped rest", + "green wedge-shaped decorative rest", + "smooth cracked green ceramic wedge", + "green ceramic-like rest with cracks", + "green rest with jagged green ridges", + "triangular green rest with uneven edge" + ], + "unseen": [ + "green and red ceramic rest", + "flat red-edged triangular rest", + "small green decorative triangular rest" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/094_rest/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/094_rest/base3.json new file mode 100644 index 00000000..11e955bc --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/094_rest/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "rest", + "seen": [ + "beige rest", + "beige curved rest", + "rest with purple base", + "oval purple base rest", + "beige rest with sculpted top", + "hand-sized uneven beige rest", + "irregularly shaped beige rest", + "rounded dark purple base rest", + "rest with gold buckle details", + "rest with golden clasp accents", + "beige top with purple oval base", + "beige textured top with purple bottom" + ], + "unseen": [ + "rest with uneven beige surface", + "compact rest for holding objects", + "small curved beige rest with base" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/095_glue/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/095_glue/base0.json new file mode 100644 index 00000000..f1cde013 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/095_glue/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "glue", + "seen": [ + "white glue bottle", + "glue with green nozzle", + "plastic adhesive bottle", + "white plastic glue container", + "glue bottle with orange label", + "smooth glue bottle with green cap", + "adhesive glue in hand-sized bottle", + "plastic glue bottle orange accents", + "compact glue bottle tapered nozzle", + "cylindrical bottle with glue inside", + "glue bottle with pointed green nozzle", + "white glue bottle orange printed label" + ], + "unseen": [ + "green cap glue bottle", + "small cylindrical glue bottle", + "green-tipped glue bottle for precision" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/095_glue/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/095_glue/base1.json new file mode 100644 index 00000000..d3c53cbe --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/095_glue/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "glue", + "seen": [ + "yellow glue bottle", + "smooth glue bottle", + "smooth adhesive bottle", + "plastic bottle for glue", + "yellow cap glue container", + "yellow bottle holding glue", + "pointed nozzle glue bottle", + "white label glue container", + "cylinder-shaped glue bottle", + "medium-sized adhesive bottle", + "yellow and orange glue bottle", + "plastic glue bottle with orange band" + ], + "unseen": [ + "bottle of glue", + "handheld plastic glue bottle", + "glue bottle with pointed tip" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/095_glue/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/095_glue/base2.json new file mode 100644 index 00000000..a800fc2b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/095_glue/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "glue", + "seen": [ + "yellow glue bottle", + "plastic glue container", + "medium size glue bottle", + "white tip of glue bottle", + "yellow cap on glue bottle", + "smooth plastic glue bottle", + "red cylindrical body of glue", + "red and white glue container", + "glue bottle with white nozzle", + "red glue bottle with white label", + "handheld cylindrical glue bottle", + "cylindrical glue bottle with lettering" + ], + "unseen": [ + "yellow flat cap", + "white glue nozzle", + "plastic glue dispenser with nozzle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/095_glue/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/095_glue/base4.json new file mode 100644 index 00000000..75e1dcfa --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/095_glue/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "glue", + "seen": [ + "school glue bottle", + "plastic glue container", + "small white glue bottle", + "hand-held glue container", + "Elmer's glue with orange cap", + "white bottle with glue inside", + "glue dispenser white and orange", + "adhesive bottle with pointed tip", + "bottle with pointed orange glue tip", + "plastic white glue bottle orange cap", + "white adhesive bottle smooth surface", + "rectangular glue bottle orange nozzle" + ], + "unseen": [ + "white glue bottle", + "glue with orange nozzle", + "smooth glue bottle with label" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/095_glue/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/095_glue/base5.json new file mode 100644 index 00000000..1e94dd22 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/095_glue/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "glue", + "seen": [ + "white glue bottle", + "hand-sized glue pack", + "small adhesive bottle", + "white glue with blue top", + "blue-tipped glue dispenser", + "smooth plastic glue bottle", + "plastic glue bottle with spout", + "glue bottle with tapered spout", + "rectangular glue tube with label", + "rectangular glue container with cap", + "blue-capped glue bottle for small jobs", + "white adhesive container labeled LOCTITE" + ], + "unseen": [ + "compact glue bottle", + "glue with blue nozzle", + "glue dispenser with vibrant blue cap" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/095_glue/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/095_glue/base6.json new file mode 100644 index 00000000..0cd6a849 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/095_glue/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "glue", + "seen": [ + "blue glue tube", + "white glue stick", + "plastic glue stick", + "adhesive glue stick", + "cylindrical glue bottle", + "pocket-sized glue stick", + "blue lid on white glue tube", + "blue and white adhesive stick", + "white tube with blue glue cap", + "small glue stick with blue top", + "smooth glue tube with blue lid", + "small cylindrical glue container" + ], + "unseen": [ + "rounded glue tube", + "compact glue stick", + "handheld blue and white glue tube" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/096_cleaner/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/096_cleaner/base0.json new file mode 100644 index 00000000..f5982878 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/096_cleaner/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "cleaner", + "seen": [ + "light blue cleaner", + "rubber-bristled cleaner", + "small cylindrical cleaner", + "palm-sized surface cleaner", + "cleaner with black bristles", + "compact cleaner for manual use", + "cleaner with rectangular button", + "smooth cleaner with flexible bristles", + "light blue cleaner with smooth texture", + "plastic cleaner with black rubber tips", + "handheld cleaner for scrubbing surfaces", + "surface cleaner with short conical bristles" + ], + "unseen": [ + "rounded cleaner for scrubbing", + "plastic cleaner with rounded edges", + "light blue body with black scrubbing tips" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/096_cleaner/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/096_cleaner/base1.json new file mode 100644 index 00000000..b732a128 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/096_cleaner/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "cleaner", + "seen": [ + "white cleaner", + "round cleaner", + "smooth cleaner", + "plastic cleaner", + "white round cleaner", + "cleaner with black bristles", + "round cleaner made of plastic", + "white cleaner with dome shape", + "white cleaner for handheld use", + "cleaner with bristles at bottom", + "round cleaner with cleaning bristles", + "white plastic cleaner with textured base" + ], + "unseen": [ + "hand-sized cleaner", + "compact cleaner with smooth top", + "palm-sized cleaner with smooth surface" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/096_cleaner/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/096_cleaner/base2.json new file mode 100644 index 00000000..a1ca86c0 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/096_cleaner/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "cleaner", + "seen": [ + "round cleaner", + "plastic cleaner", + "blue cleaner disk", + "light blue cleaner", + "medium cleaner disk", + "disk-shaped cleaner", + "smooth disk cleaner", + "smooth cleaner surface", + "round blue cleaner disk", + "light blue plastic cleaner", + "medium-sized round cleaner", + "round cleaner with smooth plastic finish" + ], + "unseen": [ + "blue disk cleaner", + "plastic disk cleaner", + "smooth light blue cleaner" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/096_cleaner/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/096_cleaner/base3.json new file mode 100644 index 00000000..c0252aa3 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/096_cleaner/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "cleaner", + "seen": [ + "rounded box cleaner", + "smooth white cleaner", + "medium-sized plastic cleaner", + "cleaner with grille on left side", + "smooth cleaner with rounded edges", + "cleaner with small holes on front", + "medium cleaner with smooth surface", + "white cleaner with rectangular shape", + "cleaner with smooth white plastic material", + "white cleaner with rounded rectangular body", + "medium cleaner with visible slots and grille", + "rounded white cleaner designed for cleaning tasks" + ], + "unseen": [ + "white cleaner", + "plastic cleaner with top slot", + "white rectangular cleaner with curved sides" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/097_screen/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/097_screen/base0.json new file mode 100644 index 00000000..e9593348 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/097_screen/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "screen", + "seen": [ + "flat screen", + "computer monitor", + "rectangular monitor", + "smooth black display", + "wide-screen black panel", + "display with a black frame", + "screen with straight edges", + "black glass-fronted monitor", + "medium-sized computer screen", + "monitor with rectangular base", + "screen with flat black surface", + "black rectangular display panel" + ], + "unseen": [ + "black screen", + "screen with a stand", + "sleek black monitor panel" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/097_screen/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/097_screen/base1.json new file mode 100644 index 00000000..f924cb5c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/097_screen/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "screen", + "seen": [ + "black screen", + "flat display screen", + "medium-sized monitor screen", + "screen resting on metal stand", + "angled screen with black border", + "vibrant lighting display screen", + "dark display with angled holder", + "screen with flat glossy surface", + "screen with bright image display", + "bordered monitor with gray support", + "rectangular screen with sturdy base", + "LED screen with black plastic frame" + ], + "unseen": [ + "screen with smooth black edges", + "monitor with smooth black border", + "bordered screen sitting on stand" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/097_screen/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/097_screen/base2.json new file mode 100644 index 00000000..a3ae084c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/097_screen/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "screen", + "seen": [ + "monitor screen", + "computer screen", + "screen on stand", + "colorful desktop screen", + "screen with visible stand", + "stand-mounted flat screen", + "desk screen with black border", + "vivid screen with black edges", + "medium screen with black frame", + "screen with sleek matte finish", + "computer screen with rectangular frame", + "black-bordered screen with display image" + ], + "unseen": [ + "screen", + "flat screen", + "rectangular monitor screen" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/097_screen/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/097_screen/base3.json new file mode 100644 index 00000000..f4368a1b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/097_screen/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "screen", + "seen": [ + "flat screen", + "smooth screen", + "black-framed screen", + "screen with black frame", + "medium-sized flat screen", + "rectangular panel screen", + "gradient rectangular screen", + "green and orange flat screen", + "screen with thin black edges", + "screen with reflective surface", + "medium screen with gradient colors", + "screen with green and orange gradient" + ], + "unseen": [ + "glass screen", + "gradient screen", + "rectangular display screen" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/098_speaker/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/098_speaker/base0.json new file mode 100644 index 00000000..be12296f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/098_speaker/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "speaker", + "seen": [ + "silver speaker", + "speaker with white top", + "speaker with circular cone", + "tabletop-sized silver speaker", + "speaker with smooth glossy finish", + "silver speaker with black rim cone", + "textured cone speaker with black rim", + "speaker with four black mounting holes", + "medium trapezoidal silver audio speaker", + "speaker cone surrounded by glossy white top", + "white-topped speaker with trapezoidal shape", + "speaker with textured center and smooth body" + ], + "unseen": [ + "trapezoidal speaker", + "medium silver speaker box", + "white cone and silver speaker" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/098_speaker/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/098_speaker/base1.json new file mode 100644 index 00000000..f38b5213 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/098_speaker/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "speaker", + "seen": [ + "black speaker", + "rounded speaker", + "medium-size black speaker", + "speaker with curved edges", + "black speaker with display on top", + "speaker with digital time display", + "compact speaker with curved edges", + "speaker with glossy black surface", + "black speaker with time shown on top", + "medium black speaker with smooth finish", + "compact black curved rectangular speaker", + "black rectangular speaker with rounded sides" + ], + "unseen": [ + "speaker with smooth black plastic", + "speaker with perforated front panel", + "speaker with mesh-covered audio output" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/098_speaker/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/098_speaker/base2.json new file mode 100644 index 00000000..d349f3ae --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/098_speaker/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "speaker", + "seen": [ + "white speaker", + "medium-sized speaker", + "compact white speaker", + "speaker with perforated sides", + "speaker with oval pattern base", + "speaker with black port on back", + "speaker with black bottom panel", + "speaker with small buttons on top", + "speaker with dual speaker grilles", + "plastic speaker with smooth finish", + "portable speaker with curved edges", + "speaker with visible button layout" + ], + "unseen": [ + "rounded speaker", + "white round-edged speaker", + "speaker with perforated top" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/098_speaker/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/098_speaker/base3.json new file mode 100644 index 00000000..cb446e24 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/098_speaker/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "speaker", + "seen": [ + "portable speaker", + "medium-sized speaker", + "smooth plastic speaker", + "speaker with flat bottom", + "speaker with smooth texture", + "speaker with ridged brown sides", + "speaker with two circular parts", + "speaker with rounded black ends", + "speaker with visible sound grill", + "speaker with black and brown mix", + "handheld speaker with rounded corners", + "rectangular speaker with rounded edges" + ], + "unseen": [ + "brown speaker", + "black and brown speaker", + "speaker with ridged black top" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/098_speaker/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/098_speaker/base4.json new file mode 100644 index 00000000..379421d1 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/098_speaker/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "speaker", + "seen": [ + "black speaker", + "smooth speaker", + "light wood speaker", + "medium tabletop speaker", + "speaker with curved edges", + "modern minimalist speaker", + "speaker with rounded corners", + "two-tone black and wood speaker", + "speaker with visible sound port", + "speaker with large black front panel", + "speaker with black sides and wood top", + "black plastic and wood finish speaker" + ], + "unseen": [ + "smooth wooden-top speaker", + "rectangular black and wood speaker", + "compact speaker with light wood accent" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/098_speaker/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/098_speaker/base5.json new file mode 100644 index 00000000..5dc869ac --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/098_speaker/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "speaker", + "seen": [ + "black speaker", + "portable speaker", + "rectangular black speaker", + "medium-sized music speaker", + "speaker with grill texture", + "small-sized portable speaker", + "speaker with rounded corners", + "speaker with neon light rings", + "speaker with smooth black body", + "speaker with glowing top lights", + "rounded-edge black rectangular speaker", + "black portable speaker with glowing top" + ], + "unseen": [ + "speaker with mesh texture", + "speaker with glowing purple circle", + "speaker with purple and blue accents" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/099_fan/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/099_fan/base0.json new file mode 100644 index 00000000..7021761c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/099_fan/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "fan", + "seen": [ + "peach fan", + "handheld peach fan", + "desk fan with peach finish", + "fan with peach grille casing", + "fan with round head and base", + "fan with copper circular blades", + "medium peach fan with round base", + "small peach fan with smooth body", + "compact peach fan with sturdy neck", + "peach-colored fan with smooth finish", + "fan with peach neck and circular base", + "fan with copper blades and peach casing" + ], + "unseen": [ + "fan with smooth peach texture", + "round peach fan with copper grille", + "tabletop peach fan with copper details" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/099_fan/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/099_fan/base1.json new file mode 100644 index 00000000..0f613749 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/099_fan/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "fan", + "seen": [ + "small handheld fan", + "clip-on light green fan", + "light green plastic fan", + "fan with protective grill", + "smooth light green air fan", + "small fan with radial blades", + "fan with smooth rounded edges", + "plastic fan with radial blades", + "circular-bladed light green fan", + "compact fan with cage-like grill", + "portable fan with clip attachment", + "clip-on fan with cylindrical base" + ], + "unseen": [ + "light green fan", + "fan with circular blades", + "cage-protected handheld fan" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/099_fan/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/099_fan/base2.json new file mode 100644 index 00000000..4c831ba7 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/099_fan/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "fan", + "seen": [ + "black fan", + "small compact fan", + "fan with grey blades", + "compact black and grey fan", + "black fan with curved stand", + "black fan with curved grips", + "medium fan with black handle", + "fan with vented circular top", + "fan with circular blade cover", + "tabletop fan with circle vents", + "handheld fan with circular vent", + "grey-bladed fan with black handle" + ], + "unseen": [ + "fan with rounded plastic frame", + "black portable fan with grey details", + "portable fan with smooth black finish" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/099_fan/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/099_fan/base3.json new file mode 100644 index 00000000..c2525bce --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/099_fan/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "fan", + "seen": [ + "white fan", + "smooth white fan", + "handheld white fan", + "compact handheld fan", + "fan with ridged grill", + "fan with circular base", + "round fan with air vents", + "medium fan with black button", + "circular fan with sturdy base", + "plastic fan with black switch", + "medium fan with smooth surface", + "white fan with circular casing" + ], + "unseen": [ + "circular fan", + "white plastic fan", + "white fan with black accents" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/099_fan/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/099_fan/base4.json new file mode 100644 index 00000000..98c5f416 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/099_fan/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "fan", + "seen": [ + "white fan", + "small fan", + "round white fan", + "portable white fan", + "smooth compact fan", + "compact plastic fan", + "fan with grid cover", + "fan with round blades", + "fan with rectangular base", + "table fan with white finish", + "white fan with adjustable arm", + "lightweight plastic adjustable fan" + ], + "unseen": [ + "plastic fan", + "white desk fan", + "fan with small round shape" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/099_fan/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/099_fan/base5.json new file mode 100644 index 00000000..df547698 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/099_fan/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "fan", + "seen": [ + "off-white fan", + "small handheld fan", + "fan with beige grilles", + "small cooling device fan", + "smooth off-white plastic fan", + "compact fan with beige blades", + "curved plastic body small fan", + "plastic fan with smooth finish", + "rounded fan with central handle", + "fan with rounded rectangular body", + "fan with cylinder-shaped handle base", + "handheld fan with circular blade housing" + ], + "unseen": [ + "beige circular blade fan", + "portable curved handle fan", + "off-white fan with beige accents" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/099_fan/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/099_fan/base6.json new file mode 100644 index 00000000..9a5c218d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/099_fan/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "fan", + "seen": [ + "pink fan", + "clip-on pink fan", + "small pink and white fan", + "clip fan with grid cover", + "fan with pink round blades", + "plastic fan with pink clip", + "pink fan with curved blades", + "compact fan with white motor", + "smooth pink fan with mesh cover", + "pink fan with white cylinder base", + "handheld pink rectangular clip fan", + "small pink fan with plastic blades" + ], + "unseen": [ + "pink fan with rectangular jaws", + "white motor base with pink fan", + "fan with round pink blade cover" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/100_seal/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/100_seal/base0.json new file mode 100644 index 00000000..c22df8d9 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/100_seal/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "seal", + "seen": [ + "red seal", + "stamp with red top", + "seal for imprinting", + "red cylindrical seal", + "red seal for stamping", + "small red stamp handle", + "seal with smooth handle", + "plastic seal with handle", + "plastic red and black seal", + "compact seal with matte base", + "hand-sized red and black seal", + "stamp with black rectangular base" + ], + "unseen": [ + "seal with black base", + "seal with smooth texture", + "red seal on rectangular base" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/100_seal/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/100_seal/base1.json new file mode 100644 index 00000000..5211b0d2 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/100_seal/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "seal", + "seen": [ + "brown seal", + "rectangular seal", + "rusty brown seal", + "hand-sized stone seal", + "seal used for imprinting", + "seal with smooth texture", + "seal with detailed script", + "seal with engraved writing", + "smooth engraved stone seal", + "seal with round bottom carving", + "engraved brown rectangular seal", + "rectangular seal for imprinting" + ], + "unseen": [ + "stone seal", + "seal with white inscriptions", + "rectangular seal with round carving" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/100_seal/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/100_seal/base2.json new file mode 100644 index 00000000..7b74c854 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/100_seal/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "seal", + "seen": [ + "small stone seal", + "reddish-brown seal", + "palm-sized stamping seal", + "seal with white engraving", + "seal made of polished stone", + "seal with carved light letters", + "seal with raised rounded edges", + "reddish seal with light markings", + "engraved reddish seal for marking", + "seal with rounded top and patterns", + "smooth textured reddish-brown seal", + "rectangular seal with decorative carvings" + ], + "unseen": [ + "reddish rectangular seal", + "smooth stone seal with writing", + "small rectangular seal for stamping" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/100_seal/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/100_seal/base3.json new file mode 100644 index 00000000..d723919d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/100_seal/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "seal", + "seen": [ + "brown seal", + "rectangular seal", + "engraved top seal", + "seal for stamping", + "compact stone seal", + "seal with sloped sides", + "seal with textured top", + "smooth rectangular seal", + "seal with smooth engraved top", + "hand-sized rectangular brown seal", + "rectangular stone seal with engraved top", + "brown rectangular seal with slanted edges" + ], + "unseen": [ + "solid brown seal", + "small slanted-edged brown seal", + "block-shaped seal with angled edges" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/100_seal/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/100_seal/base4.json new file mode 100644 index 00000000..9530de50 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/100_seal/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "seal", + "seen": [ + "brown seal", + "compact seal", + "smooth brown seal", + "seal with rounded top", + "rectangular seal block", + "brown seal for marking", + "hand-sized carved seal", + "seal with embossed patterns", + "stone seal with leaf designs", + "carved seal with leaf patterns", + "decorative brown stamping seal", + "seal showing carved detailed textures" + ], + "unseen": [ + "small carved seal", + "brown seal with leaf carvings", + "small rectangular seal for stamping" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/100_seal/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/100_seal/base6.json new file mode 100644 index 00000000..51e4e160 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/100_seal/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "seal", + "seen": [ + "yellow seal", + "seal for marking", + "yellow stamping tool", + "compact stamping seal", + "round yellow seal base", + "small seal with round base", + "seal with wooden-like grip", + "yellow seal with ridged grip", + "seal with smooth yellow bottom", + "light beige ridged handle seal", + "seal with vertical textured grip", + "handheld seal with yellow bottom" + ], + "unseen": [ + "round base seal", + "stamp with beige handle", + "seal used for pressing stamps" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/101_milk-tea/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/101_milk-tea/base0.json new file mode 100644 index 00000000..53f2fd67 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/101_milk-tea/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "milk-tea", + "seen": [ + "milk-tea cup", + "light brown milk-tea", + "printed milk-tea cup", + "spotted milk-tea cup", + "medium-sized milk-tea", + "smooth milk-tea container", + "milk-tea cup with flat top", + "milk-tea cup with brown pattern", + "milk-tea with darker brown spots", + "tapered cylindrical milk-tea cup", + "milk-tea cup with smooth texture", + "milk-tea cup made of synthetic material" + ], + "unseen": [ + "plastic milk-tea cup", + "milk-tea having flat base", + "plastic milk-tea with spotted design" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/101_milk-tea/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/101_milk-tea/base1.json new file mode 100644 index 00000000..e3db46bf --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/101_milk-tea/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "milk-tea", + "seen": [ + "milk-tea cup", + "beige drink cup", + "milk-tea cup with flat lid", + "light brown milky tea drink", + "handheld milk-tea cup with lid", + "plastic cup with tapioca pearls", + "bubble tea cup with white straw", + "boba drink cup with black pearls", + "cylindrical beige cup with straw", + "medium plastic milk-tea container", + "milk-tea with straw and round lid", + "drink cup with tapioca boba pearls" + ], + "unseen": [ + "boba milk-tea with straw", + "light beige cup for bubble tea", + "smooth beige cup with black dots" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/101_milk-tea/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/101_milk-tea/base2.json new file mode 100644 index 00000000..b5d8097c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/101_milk-tea/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "milk-tea", + "seen": [ + "milk-tea", + "brown milk-tea", + "cup of milk-tea", + "light brown tea drink", + "plastic cup of milk-tea", + "milk-tea with sealed lid", + "milk-tea in tall clear cup", + "milk-tea with chewy black pearls", + "milk-tea with dark tapioca balls", + "brown milk-tea in see-through cup", + "milk-tea with creamy brown layers", + "milk-tea in cylindrical plastic container" + ], + "unseen": [ + "milk-tea with black pearls", + "milk-tea drink inside plastic cup", + "sweet milk-tea with round black pearls" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/101_milk-tea/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/101_milk-tea/base4.json new file mode 100644 index 00000000..c9bfc4b4 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/101_milk-tea/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "milk-tea", + "seen": [ + "milk-tea cup", + "milk-tea with white straw", + "plastic milk-tea container", + "milk-tea cup with thin straw", + "tea cup with rounded white lid", + "milk-tea cup with glossy finish", + "medium-sized plastic tea container", + "milky tea inside smooth plastic cup", + "light brown tea with attached straw", + "cylindrical cup filled with milk-tea", + "white lid covering light brown drink", + "handheld milk-tea with smooth surface" + ], + "unseen": [ + "light brown tea cup", + "brown drink with white lid", + "compact milk-tea and plastic lid" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/101_milk-tea/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/101_milk-tea/base5.json new file mode 100644 index 00000000..601b3506 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/101_milk-tea/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "milk tea", + "seen": [ + "milk tea", + "light brown drink in cup", + "bubble tea with white lid", + "tall cup of brown milk tea", + "milk tea cup with red straw", + "milk tea topped with white lid", + "bubble tea cup with clear sides", + "plastic cup filled with milk tea", + "milk tea with chewy black pearls", + "hand-sized brown drink with straw", + "milk tea featuring dark tapioca balls", + "red straw sticking out of milk tea cup" + ], + "unseen": [ + "milk tea with black pearls", + "milk tea in smooth plastic cup", + "smooth light brown drink with pearls" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/101_milk-tea/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/101_milk-tea/base6.json new file mode 100644 index 00000000..e1b514f0 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/101_milk-tea/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "milk-tea", + "seen": [ + "boba milk-tea cup", + "brown and white milk-tea", + "hand-sized milk-tea drink", + "brown and white drink in cup", + "tapered lid cup with milk-tea", + "white cup filled with milk-tea", + "smooth plastic cup of milk-tea", + "milk-tea with chewy boba balls", + "plastic cup with milk-tea inside", + "milky brown drink with boba pearls", + "cylindrical plastic cup of milk-tea", + "drinkable beverage with dark boba pearls" + ], + "unseen": [ + "milk-tea cup", + "medium-sized cup of milk-tea", + "brown pearls inside sweet milk-tea" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/102_roller/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/102_roller/base0.json new file mode 100644 index 00000000..7e864802 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/102_roller/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "roller", + "seen": [ + "wood roller", + "solid wood roller", + "light brown roller", + "palm-length roller", + "medium-sized roller", + "wooden rolling tool", + "light brown wooden roller", + "roller with polished surface", + "roller for smoothing materials", + "medium roller with narrower tips", + "cylindrical roller with tapered ends", + "roller with smooth and shiny texture" + ], + "unseen": [ + "smooth wooden roller", + "roller with tapered tips", + "light brown cylindrical roller" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/102_roller/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/102_roller/base1.json new file mode 100644 index 00000000..ab099acd --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/102_roller/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "roller", + "seen": [ + "smooth wooden roller", + "small roller for dough", + "tool for flattening dough", + "roller with wooden handles", + "light brown cylinder roller", + "roller made of polished wood", + "roller with two wooden grips", + "roller with dark brown handles", + "medium-sized roller for baking", + "roller with smooth wooden cylinder", + "light brown roller for kitchen use", + "cylinder roller with rounded handles" + ], + "unseen": [ + "light brown roller", + "wooden roller with cylindrical body", + "dough roller with light wood finish" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/102_roller/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/102_roller/base2.json new file mode 100644 index 00000000..a4b87c11 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/102_roller/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "roller", + "seen": [ + "brown roller", + "long slim roller", + "palm-sized roller", + "light wood roller", + "cylindrical roller", + "roller for rolling", + "slim wooden roller", + "narrow tapered roller", + "smooth light brown roller", + "smooth cylindrical roller", + "roller with smooth texture", + "roller shaped like cylinder" + ], + "unseen": [ + "wooden roller", + "roller with tapered ends", + "light brown wooden roller" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/103_fruit/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/103_fruit/base0.json new file mode 100644 index 00000000..d3384cea --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/103_fruit/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "fruit", + "seen": [ + "green fruit", + "round fruit", + "medium round fruit", + "fruit with red stripes", + "palm-sized green fruit", + "smooth round green fruit", + "green fruit with brown stem", + "fruit with green and red mix", + "edible fruit with red blotches", + "round green fruit with dimples", + "fruit with reddish spots and stem", + "round fruit with streaky red patches" + ], + "unseen": [ + "medium fruit red and green", + "fruit with smooth green surface", + "organic green fruit with red streaks" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/103_fruit/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/103_fruit/base1.json new file mode 100644 index 00000000..e8382caf --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/103_fruit/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "fruit", + "seen": [ + "small pear-shaped fruit", + "fruit shaped like a pear", + "soft fruit with curvy body", + "smooth fruit with brown stem", + "round top tapering bottom fruit", + "stemmed yellow-green soft fruit", + "glossy yellow-green edible fruit", + "hand-sized fruit with curved shape", + "greenish yellow fruit with brown top", + "pear-shaped fruit with smooth surface", + "fruit with yellow-green shiny texture", + "hand-sized glossy greenish yellow fruit" + ], + "unseen": [ + "yellow-green fruit", + "medium-sized fruit with stem", + "pear-shaped yellow-green fruit with stem" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/103_fruit/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/103_fruit/base2.json new file mode 100644 index 00000000..24be7c5c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/103_fruit/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "fruit", + "seen": [ + "round fruit", + "yellow fruit", + "medium-sized fruit", + "smooth yellow fruit", + "glossy yellow fruit", + "bright yellow sphere fruit", + "sphere-shaped yellow fruit", + "round glossy fruit with stem", + "medium round glossy yellow fruit", + "yellow fruit suitable for eating", + "fruit with smooth and bright skin", + "perfectly round fruit with glossy skin" + ], + "unseen": [ + "yellow fruit firm and shiny", + "yellow fruit with green stem", + "palm-sized round yellow fruit" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/103_fruit/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/103_fruit/base3.json new file mode 100644 index 00000000..6d6d8ee6 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/103_fruit/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "fruit", + "seen": [ + "yellow fruit", + "medium round fruit", + "smooth yellow fruit", + "hand-sized yellow fruit", + "round bright yellow fruit", + "yellow fruit with grooves", + "sphere-shaped yellow fruit", + "yellow fruit with uneven bottom", + "round fruit with shiny yellow skin", + "solid fruit with bright yellow color", + "yellow sphere-like fruit with small bumps", + "yellow fruit with symmetrical rounded shape" + ], + "unseen": [ + "fruit with small top protrusion", + "fruit with slightly textured surface", + "fruit with bright yellow smooth skin" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/103_fruit/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/103_fruit/base4.json new file mode 100644 index 00000000..07f7d2a2 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/103_fruit/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "fruit", + "seen": [ + "pineapple", + "bumpy oval yellow fruit", + "tropical yellow pineapple", + "green-leafed yellow oval fruit", + "spiky green-topped yellow fruit", + "pineapple with rough yellow skin", + "green spiky leaves on yellow base", + "yellow fruit with green pointy top", + "rough-skinned fruit with green top", + "leaf-crowned bumpy yellow pineapple", + "yellow pineapple with green leaf spikes", + "bright yellow pineapple with green leaves" + ], + "unseen": [ + "yellow fruit", + "medium-sized tropical pineapple", + "medium oval fruit with leaf crown" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/103_fruit/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/103_fruit/base5.json new file mode 100644 index 00000000..e989767e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/103_fruit/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "fruit", + "seen": [ + "round fruit", + "round organic fruit", + "fruit with green leaf", + "red-spotted yellow fruit", + "fruit with oval green leaf", + "round fruit with small stem", + "yellow fruit with green leaf", + "yellow fruit with brown stem", + "palm-sized yellow-orange fruit", + "smooth round fruit with red blush", + "small stem on yellow-orange fruit", + "yellow-orange fruit with red spots" + ], + "unseen": [ + "smooth edible fruit", + "fruit with leaf and stem", + "smooth fruit with slight dimples" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/103_fruit/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/103_fruit/base6.json new file mode 100644 index 00000000..d7b32b09 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/103_fruit/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "fruit", + "seen": [ + "red textured fruit", + "bright red round fruit", + "fruit with green leaves", + "fruit with leaf cluster", + "small spherical red fruit", + "red fruit with brown stem", + "red fruit with rough texture", + "small round fruit with leaves", + "fruit with pointed green leaves", + "bumpy red fruit with green parts", + "bright red fruit with smooth leaves", + "palm-sized fruit with textured skin" + ], + "unseen": [ + "round fruit", + "bumpy fruit with green stem", + "round bright fruit with stem" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/104_board/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/104_board/base0.json new file mode 100644 index 00000000..d4338952 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/104_board/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "board", + "seen": [ + "board with cutout handle", + "smooth light brown board", + "board made of smooth wood", + "board for cutting or serving", + "flat brown board with cutout", + "flat smooth light brown board", + "medium rectangular wooden board", + "flat rectangular board with handle", + "rectangular board with single handle", + "light brown board with grain texture", + "medium-sized light brown wooden board", + "rectangular wooden board with handle hole" + ], + "unseen": [ + "wooden board", + "light brown board", + "light brown rectangular cutting board" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/104_board/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/104_board/base1.json new file mode 100644 index 00000000..b891931c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/104_board/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "board", + "seen": [ + "brown board", + "medium wooden board", + "dark brown cutting board", + "smooth polished brown board", + "dark brown wood serving board", + "smooth brown paddle-shaped board", + "small paddle-shaped cutting board", + "medium flat board with grip handle", + "wooden board for serving or cutting", + "dark wooden board with short handle", + "medium brown board with curved grip", + "rectangular board with rounded edges" + ], + "unseen": [ + "rounded brown wooden board", + "medium-size board for kitchen use", + "flat rectangular board with handle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/104_board/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/104_board/base2.json new file mode 100644 index 00000000..c9fc353a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/104_board/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "board", + "seen": [ + "wood board", + "flat brown board", + "brown cutting board", + "smooth surface wood board", + "medium smooth brown board", + "board with hole near bottom", + "hard wooden rectangle board", + "medium wood board with grain", + "board for cutting or serving", + "natural wood board with hole", + "board with tapered rectangle shape", + "rectangle board with grain texture" + ], + "unseen": [ + "board with angled edges", + "flat brown tapered board", + "medium-sized wooden board" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/104_board/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/104_board/base3.json new file mode 100644 index 00000000..aff234c5 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/104_board/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "board", + "seen": [ + "brown flat board", + "flat light brown board", + "smooth wooden trapezoid", + "medium flat brown panel", + "board with tapered corners", + "light brown wood trapezoid", + "light brown board with grain", + "wood board with visible grain", + "flat wooden board medium size", + "trapezoidal light brown board", + "trapezoid board with smooth surface", + "light-brown smooth trapezoidal board" + ], + "unseen": [ + "wooden board", + "medium wood panel flat shape", + "medium trapezoidal wood panel" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/104_board/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/104_board/base4.json new file mode 100644 index 00000000..728872bb --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/104_board/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "board", + "seen": [ + "medium flat board", + "rectangular orange board", + "board with slanted edges", + "plastic orange flat panel", + "smooth orange panel board", + "flat board made of plastic", + "smooth orange plastic board", + "medium orange support board", + "orange board with slanted sides", + "smooth rectangular orange board", + "orange board for holding objects", + "rectangular board with smooth surface" + ], + "unseen": [ + "orange board", + "medium-sized orange board", + "slanted rectangular orange board" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/105_sauce-can/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/105_sauce-can/base0.json new file mode 100644 index 00000000..4fcd76d4 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/105_sauce-can/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "sauce can", + "seen": [ + "sauce can", + "hand-sized sauce can", + "cylindrical sauce can", + "plastic lid sauce can", + "yellow brown sauce can", + "rounded-edge sauce can", + "sauce can with sealed top", + "white plastic lid sauce can", + "sauce can with glossy finish", + "medium sauce can with white lid", + "round sauce can with yellow pattern", + "brown and yellow cylindrical sauce can" + ], + "unseen": [ + "smooth sauce can", + "yellow label on sauce can", + "brown sauce can with top lid" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/105_sauce-can/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/105_sauce-can/base2.json new file mode 100644 index 00000000..ed594f8e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/105_sauce-can/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "sauce can", + "seen": [ + "brown-lid sauce can", + "hand-held sauce container", + "brown top white sauce can", + "white sauce can brown top", + "white sauce can with label", + "metal can with plastic lid", + "brown cap sauce storage can", + "white label brown lidded can", + "smooth metal sauce can white", + "white cylindrical sauce holder", + "cylindrical can brown round top", + "cylindrical container for sauce" + ], + "unseen": [ + "sauce can", + "medium sauce can with white label", + "medium-size cylindrical sauce can" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/105_sauce-can/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/105_sauce-can/base4.json new file mode 100644 index 00000000..7e3eb51d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/105_sauce-can/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "sauce can", + "seen": [ + "brown sauce can", + "metal sauce can", + "medium sauce can", + "brown top sauce can", + "cylindrical sauce can", + "yellow label sauce can", + "printed label sauce can", + "sauce can with brown lid", + "sauce can for condiments", + "yellow and brown sauce can", + "smooth cylindrical sauce can", + "sauce can with ingredient information" + ], + "unseen": [ + "hand-sized sauce can", + "metal body sauce can", + "sauce can with smooth surface" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/105_sauce-can/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/105_sauce-can/base5.json new file mode 100644 index 00000000..42fc8e42 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/105_sauce-can/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "sauce can", + "seen": [ + "red sauce can", + "smooth sauce can", + "cylindrical sauce can", + "red plastic sauce can", + "red and gold sauce can", + "sauce can with flat top", + "red cylindrical sauce can", + "sauce can with flat bottom", + "medium-sized red sauce can", + "red can with gold patterns", + "sauce can with white details", + "red sauce can for holding condiments" + ], + "unseen": [ + "medium sauce can", + "sauce can with red lid", + "smooth cylindrical sauce can" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/105_sauce-can/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/105_sauce-can/base6.json new file mode 100644 index 00000000..1fb02f12 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/105_sauce-can/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "sauce can", + "seen": [ + "smooth sauce can", + "metal body sauce can", + "cylindrical sauce can", + "sauce can with red lid", + "red and orange sauce can", + "red can with orange label", + "small cylindrical sauce can", + "small metal sauce container", + "sauce can with printed design", + "sauce can with rounded top cap", + "compact sauce can with plastic lid", + "sturdy sauce can with glossy finish" + ], + "unseen": [ + "red sauce can", + "red can with white text", + "sauce can with round ridged lid" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/106_skillet/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/106_skillet/base0.json new file mode 100644 index 00000000..c1679b57 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/106_skillet/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "skillet", + "seen": [ + "metal skillet", + "dark gray skillet", + "flat-bottom skillet", + "skillet for frying food", + "smooth non-stick skillet", + "round skillet with handle", + "medium-sized frying skillet", + "gray skillet for stovetop cooking", + "skillet with black cooking surface", + "round skillet with open handle end", + "metal skillet with comfortable grip", + "black-surfaced skillet with gray sides" + ], + "unseen": [ + "dark skillet with shiny finish", + "pan-shaped skillet with hole in handle", + "versatile skillet for frying and sauteing" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/106_skillet/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/106_skillet/base1.json new file mode 100644 index 00000000..270039da --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/106_skillet/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "skillet", + "seen": [ + "round cooking pan", + "dark pan with sturdy grip", + "black pan with curved edges", + "heavy black cooking skillet", + "medium black pan with handle", + "solid dark-colored round pan", + "round pan with hole in handle", + "smooth round pan made of iron", + "black skillet with smooth base", + "skillet with extended grip handle", + "flat-bottomed skillet with long handle", + "versatile skillet for stovetop cooking" + ], + "unseen": [ + "black skillet", + "round cast-iron skillet", + "iron skillet for frying foods" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/106_skillet/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/106_skillet/base2.json new file mode 100644 index 00000000..a2f89828 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/106_skillet/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "skillet", + "seen": [ + "black skillet", + "heavy cast iron skillet", + "pan-shaped black skillet", + "black cooking tool skillet", + "skillet with looped handle", + "skillet with wide round base", + "skillet with sturdy material", + "black skillet with long handle", + "black skillet for frying foods", + "black iron skillet for cooking", + "medium skillet with rough outside", + "flat-bottomed black cast iron skillet" + ], + "unseen": [ + "round black skillet", + "smooth interior skillet", + "medium black frying skillet" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/106_skillet/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/106_skillet/base3.json new file mode 100644 index 00000000..62ac4769 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/106_skillet/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "skillet", + "seen": [ + "gray pan", + "medium skillet", + "dark gray skillet", + "skillet for cooking", + "hefty black skillet", + "pan with dark finish", + "smooth cast iron skillet", + "round skillet with handle", + "smooth round cooking skillet", + "gray cooking pan with handle", + "medium round cast iron skillet", + "rounded skillet with a holey handle" + ], + "unseen": [ + "dark gray round cast iron pan", + "blackish-gray cast iron skillet", + "cooking skillet with grippy handle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/107_soap/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/107_soap/base0.json new file mode 100644 index 00000000..63d093fe --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/107_soap/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "soap", + "seen": [ + "yellow soap", + "hand-sized soap", + "rectangular soap", + "solid yellow soap bar", + "soap bar in pale yellow", + "soap with rounded corners", + "solid soap in light yellow", + "palm-sized rectangular soap", + "yellow soap bar for cleaning", + "yellow soap with smooth surface", + "smooth yellow rectangular soap bar", + "rectangular soap with rounded edges" + ], + "unseen": [ + "smooth soap", + "light yellow cleaning soap", + "light yellow soap for hygiene" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/107_soap/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/107_soap/base1.json new file mode 100644 index 00000000..53198d78 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/107_soap/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "soap", + "seen": [ + "soap bar", + "gray soap", + "light gray soap bar", + "solid rectangular soap", + "smooth gray bar of soap", + "slanted rectangular soap", + "small solid gray cleaning soap", + "slanted soap with smooth surface", + "light gray soap with angled ends", + "light gray rectangular cleaning soap", + "gray rectangular soap with angled edges", + "rectangular soap with slight groove pattern" + ], + "unseen": [ + "hand-sized gray soap", + "soap with smooth grooves", + "solid gray rectangular cleaning soap" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/107_soap/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/107_soap/base2.json new file mode 100644 index 00000000..d607b10e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/107_soap/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "soap", + "seen": [ + "blue soap", + "smooth soap", + "small blue soap", + "soap with smooth edges", + "compact blue bar of soap", + "solid blue cleansing soap", + "blue soap for hand washing", + "hand-sized rectangular soap", + "bright blue smooth soap bar", + "soap bar with light blue shade", + "rounded rectangular blue soap bar", + "rectangular soap with rounded corners" + ], + "unseen": [ + "bright blue soap bar", + "soap bar shaped rectangular", + "blue soap with printed patterns" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/107_soap/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/107_soap/base3.json new file mode 100644 index 00000000..9620f73f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/107_soap/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "soap", + "seen": [ + "dark soap bar", + "grainy brown soap", + "soap with tan specks", + "rounded bar of dark soap", + "small dark cleaning soap", + "solid brown cleaning soap", + "dark brown hand-sized soap", + "soap bar with curved edges", + "soap with uneven tan grains", + "smooth soap with grainy spots", + "cleaning soap with spotted texture", + "rectangular soap with rounded edges" + ], + "unseen": [ + "brown soap", + "dark bar soap", + "skincare soap bar" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/108_block/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/108_block/base0.json new file mode 100644 index 00000000..29307aa5 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/108_block/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "108 block", + "seen": [ + "108 block", + "brown block", + "brown log-like block", + "block with tapered ends", + "block with rounded ends", + "smooth block for stacking", + "palm-sized brown 108 block", + "brown cylinder-shaped block", + "smooth polished brown block", + "rounded brown cylinder block", + "smooth block used for stacking", + "rounded cylinder block with tapered edges" + ], + "unseen": [ + "medium-sized rounded block", + "cylindrical block for modeling", + "brown medium-size cylinder block" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/108_block/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/108_block/base1.json new file mode 100644 index 00000000..28a7bd8d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/108_block/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "108 block", + "seen": [ + "orange and blue block", + "cylindrical 108 block", + "smooth patterned cylinder", + "patterned medium-sized block", + "medium cylinder with patterns", + "solid block with curved sides", + "rounded block with blue stripe", + "smooth cylindrical orange block", + "orange and blue simulation block", + "cylinder with blue center stripe", + "color-blocked orange and blue cylinder", + "cylindrical object with orange and blue" + ], + "unseen": [ + "rounded block", + "cylinder with orange top", + "orange cylinder with smooth edges" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/108_block/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/108_block/base2.json new file mode 100644 index 00000000..0be7506c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/108_block/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "108 block", + "seen": [ + "light blue cylinder", + "tube-shaped 108 block", + "medium-sized blue tube", + "yellow-ended blue block", + "smooth cylindrical block", + "plastic smooth blue block", + "blue tube with yellow ends", + "108 block with smooth surface", + "yellow-capped cylindrical block", + "cylindrical block with yellow ends", + "plastic blue tube with yellow rings", + "light blue cylinder with yellow edges" + ], + "unseen": [ + "blue cylinder with yellow rims", + "blue cylinder capped with yellow", + "medium blue cylinder with yellow caps" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/108_block/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/108_block/base3.json new file mode 100644 index 00000000..5e033a36 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/108_block/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "block", + "seen": [ + "striped block", + "smooth plastic block", + "cylindrical toy block", + "medium-sized toy cylinder", + "striped block with red ends", + "toy with smooth plastic finish", + "colorful striped cylinder block", + "cylinder with smile and stripes", + "block with cheerful face on top", + "toy block with smiley face design", + "plastic block with colorful bands", + "yellow red blue striped barrel-shaped block" + ], + "unseen": [ + "medium toy cylinder for stacking", + "block with red blue yellow stripes", + "play block with rounded cylinder shape" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/108_block/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/108_block/base4.json new file mode 100644 index 00000000..e9b9a549 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/108_block/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "block", + "seen": [ + "smooth yellow block", + "plastic yellow block", + "handheld yellow block", + "medium-sized yellow block", + "hard plastic yellow block", + "yellow block with rounded shape", + "cylindrical block with tapered ends", + "block with yellow color and grooves", + "yellow block with grooves on one end", + "bright yellow block with smooth texture", + "yellow block showing grooves at one side", + "hand-sized yellow block with smooth finish" + ], + "unseen": [ + "yellow block", + "yellow cylindrical block", + "yellow block made of durable plastic" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/108_block/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/108_block/base5.json new file mode 100644 index 00000000..f4a8d081 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/108_block/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "block", + "seen": [ + "orange block", + "plastic cylinder block", + "block with white stripes", + "block with circular ends", + "striped cylindrical block", + "smooth orange cylinder block", + "orange block with white bands", + "medium round block with stripes", + "smooth shiny orange and white block", + "block with orange and white sections", + "solid orange block with white accents", + "block featuring alternating orange and white design" + ], + "unseen": [ + "round block", + "orange and white medium block", + "block with distinct colored pattern" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/108_block/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/108_block/base6.json new file mode 100644 index 00000000..5b376a41 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/108_block/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "108 block", + "seen": [ + "108 block with stripes", + "108 block with capped end", + "striped yellow and cyan cylinder", + "yellow cylinder with cyan stripes", + "108 block with alternating colors", + "block with circular design on top", + "decorative block with smooth finish", + "medium-sized cylinder with white cap", + "yellow 108 block with glossy surface", + "block with white circular feature on top", + "cylinder with yellow base and cyan stripes", + "smooth plastic cylinder with yellow accents" + ], + "unseen": [ + "cylindrical 108 block", + "yellow and cyan block", + "cyan-striped yellow cylinder" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/109_hydrating-oil/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/109_hydrating-oil/base0.json new file mode 100644 index 00000000..ae77ad2f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/109_hydrating-oil/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "hydrating-oil", + "seen": [ + "yellow hydrating oil bottle", + "small hydrating oil container", + "hydrating oil bottle in plastic", + "hydrating oil tube with round body", + "tapered neck yellow bottle for oil", + "small yellow cylindrical oil bottle", + "bright yellow bottle with white nozzle", + "rounded yellow bottle for skincare oil", + "hydrating oil bottle with sleek design", + "smooth yellow bottle with oil dispenser", + "plastic yellow bottle with smooth surface", + "white-capped yellow bottle for hydrating oil" + ], + "unseen": [ + "yellow bottle for hydrating oil", + "hydrating oil bottle with white cap", + "yellow bottle with pointed white nozzle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/109_hydrating-oil/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/109_hydrating-oil/base1.json new file mode 100644 index 00000000..415bb439 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/109_hydrating-oil/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "hydrating oil", + "seen": [ + "small orange bottle", + "hydrating oil bottle", + "glass bottle with beige cap", + "bottle with dark red dropper", + "hydrating oil bottle with red label", + "smooth orange bottle with dropper top", + "compact bottle with beige plastic cap", + "small white-tinted orange glass bottle", + "orange and white hydrating oil container", + "small cylindrical bottle for hydrating oil", + "hydrating oil bottle with ridged beige cap", + "orange hydrating oil bottle with glass body" + ], + "unseen": [ + "bottle with dark red suction dropper", + "hydrating oil container with rubber dropper", + "cylindrical bottle for storing hydrating oil" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/109_hydrating-oil/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/109_hydrating-oil/base2.json new file mode 100644 index 00000000..3f3bccc7 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/109_hydrating-oil/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "hydrating-oil", + "seen": [ + "hydrating-oil container", + "yellow hydrating-oil bottle", + "hydrating-oil with pump top", + "bright yellow hydrating-oil", + "hydrating-oil pump dispenser", + "hydrating-oil glossy surface", + "hydrating-oil pump head white", + "plastic hydrating-oil container", + "smooth yellow hydrating-oil bottle", + "hydrating-oil bottle with black text", + "hydrating-oil bright yellow cylinder", + "hydrating-oil moisturizing oil inside" + ], + "unseen": [ + "medium yellow hydrating-oil", + "hydrating-oil pump mechanism", + "hydrating-oil bottle fits in hand" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/109_hydrating-oil/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/109_hydrating-oil/base5.json new file mode 100644 index 00000000..282cebd1 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/109_hydrating-oil/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "hydrating oil", + "seen": [ + "hydrating oil pump bottle", + "orange hydrating oil bottle", + "hydrating oil with red pump", + "handheld orange pump bottle", + "smooth plastic oil container", + "medium orange bottle with oil", + "pump bottle for hydrating oil", + "red pump on orange oil bottle", + "plastic hydrating oil dispenser", + "orange bottle with rounded edges", + "red nozzle on orange oil container", + "orange bottle with bright red pump" + ], + "unseen": [ + "hydrating oil in orange packaging", + "orange bottle holding hydrating oil", + "rectangular orange bottle with pump" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/110_basket/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/110_basket/base0.json new file mode 100644 index 00000000..b12edb32 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/110_basket/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "basket", + "seen": [ + "brown basket", + "woven basket", + "basket with handle", + "light brown oval basket", + "basket with loop handle", + "basket for holding items", + "basket made of woven wood", + "basket for carrying things", + "brown basket with woven ridges", + "wooden basket with curved handle", + "smooth ribbed light brown basket", + "oval-shaped basket with woven body" + ], + "unseen": [ + "basket with rounded edges", + "medium basket with smooth texture", + "oval basket with medium storage space" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/110_basket/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/110_basket/base1.json new file mode 100644 index 00000000..97cde3c5 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/110_basket/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "basket", + "seen": [ + "yellow basket", + "plastic basket", + "handheld basket", + "bright yellow basket", + "basket with black handle", + "rectangular yellow basket", + "basket with perforated sides", + "basket with arched black grip", + "medium yellow basket with slots", + "yellow basket for carrying stuff", + "rectangular basket with smooth surface", + "carrying basket with black plastic handle" + ], + "unseen": [ + "yellow basket with open slots", + "yellow rectangular basket with dual grips", + "medium-sized basket with perforated texture" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/110_basket/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/110_basket/base2.json new file mode 100644 index 00000000..1ea37486 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/110_basket/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "basket", + "seen": [ + "red basket", + "plastic basket", + "shopping basket", + "basket for carrying groceries", + "rectangular red plastic basket", + "smooth basket with mesh design", + "lightweight basket for easy carrying", + "red plastic basket with curved handle", + "medium rectangular basket with open top", + "rectangular basket with thin metal handle", + "medium basket with open rectangular shape", + "shopping basket with open top and mesh sides" + ], + "unseen": [ + "basket with gray handle", + "bright red basket with mesh holes", + "red basket with sturdy metal handle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/110_basket/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/110_basket/base3.json new file mode 100644 index 00000000..3f3b8c21 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/110_basket/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "basket", + "seen": [ + "white basket", + "white basket with green trim", + "basket with smooth white body", + "plastic basket with metal edge", + "medium basket with green handle", + "white basket for carrying items", + "basket with sturdy metal handle", + "basket with mesh pattern and bar", + "basket with orange-striped handle", + "rectangular basket with mesh holes", + "plastic basket with curved metal bar", + "rectangular basket with perforated sides" + ], + "unseen": [ + "hand-sized basket with open design", + "green handle basket with orange stripe", + "rectangular basket with a sturdy handle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/111_callbell/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/111_callbell/base0.json new file mode 100644 index 00000000..391c3ab4 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/111_callbell/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "callbell", + "seen": [ + "handheld callbell", + "black handle callbell", + "hand-operated callbell", + "golden bell with wooden grip", + "small gold and black callbell", + "gold callbell with curved base", + "rounded bell with black handle", + "metallic bell with wooden handle", + "cylindrical black handle callbell", + "lightweight bell with black handle", + "compact callbell with glossy finish", + "gold callbell with smooth black grip" + ], + "unseen": [ + "gold callbell", + "shiny gold bell part", + "gold callbell for signaling" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/111_callbell/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/111_callbell/base1.json new file mode 100644 index 00000000..96655ff5 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/111_callbell/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "callbell", + "seen": [ + "golden callbell", + "small callbell with handle", + "yellow bell with long handle", + "handlebell with golden finish", + "bell-shaped callbell with grip", + "smooth golden bell with handle", + "callbell with sleek shiny design", + "thin handle gold-colored callbell", + "compact callbell with shiny texture", + "ringing callbell with metallic look", + "callbell with smooth polished surface", + "attention-grabbing golden ringing callbell" + ], + "unseen": [ + "metal callbell", + "bright yellow handheld callbell", + "golden bell for attention calling" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/111_callbell/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/111_callbell/base2.json new file mode 100644 index 00000000..871856e8 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/111_callbell/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "callbell", + "seen": [ + "medium-sized callbell", + "smooth metallic callbell", + "golden and beige callbell", + "medium shiny golden callbell", + "callbell with golden bell head", + "callbell with shiny golden head", + "medium callbell with forked handle", + "handheld callbell with smooth bell", + "callbell for ringing and attention", + "callbell made of metal and plastic", + "bell-shaped callbell with beige handle", + "flared-bottom callbell with forked grip" + ], + "unseen": [ + "golden callbell", + "shiny golden bell-shaped callbell", + "callbell with beige forked handle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/111_callbell/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/111_callbell/base3.json new file mode 100644 index 00000000..654c96ba --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/111_callbell/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "callbell", + "seen": [ + "polished yellow callbell", + "yellow and black callbell", + "yellow bell-shaped object", + "small bell with black handle", + "yellow dome with black stick", + "callbell with smooth exterior", + "metal bell with plastic handle", + "yellow bell with shiny clapper", + "curved bell shape with clapper", + "metal callbell with black grip", + "tiny callbell for ringing sound", + "yellow callbell with silver inside" + ], + "unseen": [ + "small ringing callbell", + "handheld yellow callbell", + "black plastic handle with bell top" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/111_callbell/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/111_callbell/base4.json new file mode 100644 index 00000000..b3285812 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/111_callbell/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "callbell", + "seen": [ + "golden callbell", + "bell with wooden grip", + "shiny golden callbell", + "wooden callbell handle", + "medium handheld callbell", + "smooth shiny golden bell", + "callbell for making sound", + "cone-shaped golden callbell", + "light wooden handle callbell", + "medium-sized golden callbell", + "callbell with cone-shaped bell", + "golden bell with smooth handle" + ], + "unseen": [ + "wooden handle callbell", + "alert-sounding callbell", + "metallic bell with wooden grip" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/111_callbell/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/111_callbell/base5.json new file mode 100644 index 00000000..a03d47b3 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/111_callbell/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "callbell", + "seen": [ + "brown callbell", + "wooden callbell", + "dark brown callbell", + "smooth brown callbell", + "cone-shaped metal bell", + "bell with long wooden grip", + "small bell with wooden handle", + "callbell with cone-shaped top", + "brown bell with smooth surface", + "rounded bell with ringable part", + "callbell with curved wooden handle", + "compact bell for calling attention" + ], + "unseen": [ + "hand-sized callbell", + "hand bell with polished texture", + "classic wooden and metal callbell" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/112_tea-box/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/112_tea-box/base0.json new file mode 100644 index 00000000..b519c856 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/112_tea-box/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tea box", + "seen": [ + "green tea box", + "green box with tea label", + "palm-sized tea container", + "printed tea box with lid", + "tea box with beige edges", + "small rectangular tea box", + "tea box with leafy design", + "beige-edged green tea box", + "smooth tea box with print", + "cardboard tea box with logo", + "light green cardboard tea box", + "leaf-patterned green tea storage" + ], + "unseen": [ + "compact tea box", + "rectangular green box for tea", + "box with green leafy patterns" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/112_tea-box/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/112_tea-box/base1.json new file mode 100644 index 00000000..1d3b12bf --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/112_tea-box/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tea-box", + "seen": [ + "rectangular tea-box", + "box for holding tea", + "smooth glossy tea-box", + "glossy bright tea-box", + "red cardboard tea-box", + "rectangular box for tea", + "medium-sized red tea-box", + "red cuboid-shaped tea-box", + "tea-box with white TEA text", + "bright red tea-box with logo", + "tea-box with rectangular edges", + "tea-box with bold white writing" + ], + "unseen": [ + "red tea-box", + "medium tea-box for gatherings", + "red tea-box with golden designs" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/112_tea-box/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/112_tea-box/base2.json new file mode 100644 index 00000000..1e9d8606 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/112_tea-box/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tea-box", + "seen": [ + "tea-box", + "black tea-box", + "printed tea-box", + "rectangle tea-box", + "cardboard tea-box", + "decorative tea-box", + "smooth black tea-box", + "gold accented tea-box", + "tea-box with floral design", + "compact rectangular tea-box", + "black tea-box with gold logo", + "black tea-box with turquoise sides" + ], + "unseen": [ + "small tea-box", + "turquoise and black tea-box", + "black tea-box with golden details" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/112_tea-box/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/112_tea-box/base3.json new file mode 100644 index 00000000..c71b60d4 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/112_tea-box/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tea box", + "seen": [ + "yellow tea box", + "golden tea box", + "cuboid tea box", + "yellow box for tea", + "light yellow tea box", + "small cuboid tea box", + "square-shaped tea box", + "compact golden tea box", + "rectangular yellow tea box", + "tea box with floral designs", + "tea box with printed leaves", + "yellow tea box with smooth surface" + ], + "unseen": [ + "small tea box", + "decorative golden tea box", + "tea box with green leaves" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/112_tea-box/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/112_tea-box/base4.json new file mode 100644 index 00000000..9d904163 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/112_tea-box/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tea-box", + "seen": [ + "tea-box", + "smooth tea-box", + "plastic tea-box", + "compact tea-box", + "light beige tea-box", + "tea-box with top lid", + "hard plastic tea-box", + "square-shaped tea-box", + "light beige box for tea", + "small rectangular tea-box", + "beige tea-box with handle", + "tea-box with curved edges" + ], + "unseen": [ + "glossy tea-box", + "tea-box with small handle", + "tea-box with smooth glossy finish" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/112_tea-box/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/112_tea-box/base5.json new file mode 100644 index 00000000..32ea6851 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/112_tea-box/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "tea box", + "seen": [ + "tea box", + "box for tea storage", + "compact beige box for tea", + "gold-decorated beige tea box", + "cube tea box with gold designs", + "small box with golden patterns", + "small cube-shaped tea container", + "small tea box with gold patterns", + "cube tea box with embossed designs", + "beige cube with golden leaf patterns", + "smooth cube with golden leaf designs", + "light beige box with embossed gold leaf" + ], + "unseen": [ + "light beige cube", + "beige box with golden decoration", + "cube tea box with shiny embossed patterns" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/113_coffee-box/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/113_coffee-box/base0.json new file mode 100644 index 00000000..889f6a53 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/113_coffee-box/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "coffee-box", + "seen": [ + "brown coffee-box", + "rectangular coffee-box", + "medium coffee-box for storage", + "medium rectangular coffee-box", + "coffee-box with smooth cardboard", + "coffee-box with cup image on top", + "light brown box labeled for coffee", + "printed coffee-box for coffee storage", + "light brown coffee-box with printed text", + "medium light brown coffee-box for coffee", + "rectangular coffee-box with smooth texture", + "cardboard coffee-box with light brown color" + ], + "unseen": [ + "smooth brown coffee-box", + "coffee-box with logo on top", + "coffee-box featuring cup and beans image" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/113_coffee-box/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/113_coffee-box/base1.json new file mode 100644 index 00000000..69dbcd52 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/113_coffee-box/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "coffee-box", + "seen": [ + "light brown coffee-box", + "rectangular coffee-box", + "medium-sized coffee-box", + "coffee-box with green logo", + "smooth cardboard coffee-box", + "medium box with flat surfaces", + "coffee-box with Starbucks logo", + "flat coffee-box with smooth sides", + "light brown box with green emblem", + "coffee-box having green round logo", + "Starbucks coffee-box with smooth texture", + "medium rectangular coffee-box with handle" + ], + "unseen": [ + "printed light brown coffee-box", + "rectangular brown box for coffee", + "cardboard box for storing coffee" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/113_coffee-box/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/113_coffee-box/base2.json new file mode 100644 index 00000000..56909699 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/113_coffee-box/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "coffee-box", + "seen": [ + "rectangular coffee-box", + "medium-sized coffee-box", + "coffee-box with flat top", + "compact white coffee-box", + "cardboard-like coffee-box", + "coffee-box with flat sides", + "orange and white coffee-box", + "box with orange coffee label", + "coffee-box with smooth finish", + "white box with coffee branding", + "hand-sized rectangular coffee-box", + "coffee-box with white and orange design" + ], + "unseen": [ + "white coffee-box", + "smooth textured coffee-box", + "coffee-box with orange circle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/113_coffee-box/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/113_coffee-box/base3.json new file mode 100644 index 00000000..d3e63420 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/113_coffee-box/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "coffee box", + "seen": [ + "coffee box", + "brown coffee box", + "rectangular coffee box", + "box for coffee storage", + "medium-sized coffee box", + "brown box holding coffee", + "cardboard box for coffee", + "box with coffee cup label", + "coffee box with printed design", + "light brown cardboard coffee box", + "light brown box with coffee image", + "light brown coffee storage container" + ], + "unseen": [ + "smooth coffee box", + "coffee box with slanted sides", + "coffee box with dark brown top" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/113_coffee-box/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/113_coffee-box/base4.json new file mode 100644 index 00000000..68488fc8 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/113_coffee-box/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "coffee box", + "seen": [ + "light brown coffee box", + "rectangular coffee box", + "medium-sized coffee box", + "coffee box with cup logo", + "light cardboard coffee box", + "box with coffee logo on top", + "smooth light brown coffee box", + "brown box with coffee cup image", + "medium coffee box with sealed top", + "flat rectangular box labeled coffee", + "smooth rectangular brown coffee box", + "medium light coffee box for holding" + ], + "unseen": [ + "sealed coffee box with flat sides", + "light brown box for coffee storage", + "rectangular storage box for coffee" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/113_coffee-box/base5.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/113_coffee-box/base5.json new file mode 100644 index 00000000..5df24f1d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/113_coffee-box/base5.json @@ -0,0 +1,22 @@ +{ + "raw_description": "coffee-box", + "seen": [ + "red coffee-box", + "bright red coffee-box", + "coffee-box with sachets", + "smooth glossy coffee-box", + "rectangular red coffee-box", + "red box for coffee sachets", + "coffee-box with smooth finish", + "coffee-box with Nescafe branding", + "medium rectangular red coffee-box", + "coffee-box with creamy coffee image", + "Nescafe coffee-box with shiny surface", + "rectangular box holding coffee sachets" + ], + "unseen": [ + "medium coffee-box", + "cardboard coffee-box", + "box with coffee graphics" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/113_coffee-box/base6.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/113_coffee-box/base6.json new file mode 100644 index 00000000..ad20ff4e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/113_coffee-box/base6.json @@ -0,0 +1,22 @@ +{ + "raw_description": "coffee-box", + "seen": [ + "medium coffee-box", + "smooth coffee-box", + "packaging coffee-box", + "cardboard coffee-box", + "light brown coffee-box", + "rectangular coffee-box", + "coffee-box with angled top", + "light brown box for coffee", + "orange and brown coffee-box", + "coffee-box with orange logo", + "coffee-box with slanted side", + "angled rectangular light brown coffee-box" + ], + "unseen": [ + "coffee-box with mug image", + "box with coffee mug design", + "coffee-box rectangular with smooth texture" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/114_bottle/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/114_bottle/base1.json new file mode 100644 index 00000000..cada5c40 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/114_bottle/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "yellow bottle", + "bottle with green cap", + "bottle made of smooth plastic", + "medium yellow cylindrical bottle", + "yellow bottle with smooth texture", + "green-striped cap on yellow bottle", + "bottle with small ridged green cap", + "smooth yellow bottle with ridged cap", + "yellow bottle body with tapered neck", + "yellow liquid container with green top", + "medium plastic bottle with tapered neck", + "cylindrical yellow bottle for holding liquids" + ], + "unseen": [ + "yellow plastic bottle", + "plastic bottle with two distinct parts", + "yellow bottle with elongated cylinder body" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/114_bottle/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/114_bottle/base2.json new file mode 100644 index 00000000..b732385e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/114_bottle/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "hand-sized bottle", + "transparent bottle", + "cylindrical bottle", + "bottle with narrow neck", + "medium-sized water bottle", + "bottle with turquoise cap", + "bottle with ribbed design", + "bottle for holding liquids", + "clear bottle with blue sections", + "water bottle with textured body", + "blue-accented cylindrical bottle", + "smooth transparent plastic bottle" + ], + "unseen": [ + "plastic bottle", + "bottle with ridges", + "bottle with blue highlights" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/114_bottle/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/114_bottle/base3.json new file mode 100644 index 00000000..a530ff83 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/114_bottle/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "red bottle", + "black and red bottle", + "smooth plastic bottle", + "red-labeled soda bottle", + "hand-sized drinking bottle", + "plastic bottle with red cap", + "black bottle with red label", + "drink bottle with slim neck", + "bottle with curved body shape", + "red and black beverage bottle", + "rounded bottle with a white neck", + "black soda bottle with white top" + ], + "unseen": [ + "medium black bottle", + "bottle with red screw cap", + "plastic bottle with ridges near bottom" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/114_bottle/base4.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/114_bottle/base4.json new file mode 100644 index 00000000..89150468 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/114_bottle/base4.json @@ -0,0 +1,22 @@ +{ + "raw_description": "bottle", + "seen": [ + "transparent bottle", + "blue capped bottle", + "bottle with blue cap", + "bottle with ridged sides", + "plastic bottle with label", + "medium bottle for liquids", + "blue label and blue cap bottle", + "clear body bottle with blue top", + "handheld transparent water bottle", + "bottle with tapered neck and body", + "medium bottle with blue and clear parts", + "bottle with ridged grip and smooth neck" + ], + "unseen": [ + "cylindrical plastic bottle", + "plastic bottle with smooth finish", + "transparent bottle with label around middle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/115_perfume/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/115_perfume/base0.json new file mode 100644 index 00000000..f3aecad3 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/115_perfume/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "perfume", + "seen": [ + "perfume bottle", + "pink perfume bottle", + "glossy pink glass bottle", + "palm-sized perfume bottle", + "small fragrance container", + "small glass perfume bottle", + "bottle with smooth plastic cap", + "pink perfume with white design", + "handheld glossy perfume bottle", + "glass perfume bottle with pink cap", + "light pink perfume with white accents", + "rectangular pink perfume bottle with cap" + ], + "unseen": [ + "perfume bottle with rounded edges", + "light pink bottle with spray nozzle", + "pink rectangular bottle for fragrance" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/115_perfume/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/115_perfume/base1.json new file mode 100644 index 00000000..2495dd02 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/115_perfume/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "perfume", + "seen": [ + "grey perfume bottle", + "compact perfume bottle", + "small shiny perfume bottle", + "grey bottle with blue design", + "perfume bottle with shiny cap", + "smooth grey perfume container", + "handheld perfume spray bottle", + "glossy grey bottle for perfume", + "bottle with spray for fragrance", + "grey bottle with cylindrical cap", + "blue-streaked grey perfume bottle", + "small fragrance bottle with metallic cap" + ], + "unseen": [ + "curved rectangular fragrance bottle", + "curved rectangular grey perfume bottle", + "plastic perfume bottle with curved edges" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/115_perfume/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/115_perfume/base2.json new file mode 100644 index 00000000..f8afbeb1 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/115_perfume/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "perfume", + "seen": [ + "blue perfume bottle", + "small bright blue bottle", + "smooth bright blue perfume", + "blue perfume for fragrance", + "blue perfume with black top", + "glass bottle with black lid", + "rectangular blue glass perfume", + "bottle with glossy blue finish", + "bright blue fragrance container", + "rectangular blue perfume bottle", + "fragrance bottle with black cap", + "blue perfume bottle with black cap" + ], + "unseen": [ + "small glossy blue bottle", + "blue glass perfume holder", + "handheld rectangular perfume bottle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/115_perfume/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/115_perfume/base3.json new file mode 100644 index 00000000..67cb0281 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/115_perfume/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "perfume", + "seen": [ + "perfume bottle", + "small perfume bottle", + "perfume bottle with bow", + "Miss Dior perfume bottle", + "palm-sized perfume bottle", + "decorative perfume bottle", + "transparent perfume bottle", + "fragrance bottle Miss Dior", + "smooth glass perfume bottle", + "perfume bottle with spray nozzle", + "small rectangular perfume bottle", + "perfume with transparent liquid inside" + ], + "unseen": [ + "glass perfume bottle", + "light pink perfume bottle", + "rectangular perfume bottle" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/116_keyboard/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/116_keyboard/base0.json new file mode 100644 index 00000000..a1cba8d8 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/116_keyboard/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "keyboard", + "seen": [ + "keyboard", + "plastic body keyboard", + "multi-colored keyboard", + "keyboard with black base", + "rectangular desk keyboard", + "rectangular typing keyboard", + "medium keyboard fits on desk", + "multi-colored keys on keyboard", + "keyboard for typing and gaming", + "keyboard with colorful keycaps", + "keyboard with smooth textured keys", + "tilted rectangular typing keyboard" + ], + "unseen": [ + "keyboard with 116 keys", + "keyboard with plastic base", + "matte black keyboard with color keys" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/116_keyboard/base1.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/116_keyboard/base1.json new file mode 100644 index 00000000..88f3415f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/116_keyboard/base1.json @@ -0,0 +1,22 @@ +{ + "raw_description": "keyboard", + "seen": [ + "black keyboard", + "rectangular keyboard", + "keyboard made of plastic", + "keyboard with tapered edges", + "smooth black typing keyboard", + "rectangular keyboard for typing", + "flat black rectangular keyboard", + "keyboard with textured black keys", + "keyboard with black plastic frame", + "medium-sized black typing keyboard", + "black keyboard with smooth surface", + "plastic rectangular computer keyboard" + ], + "unseen": [ + "desk-sized black keyboard", + "keyboard for desktop computers", + "keyboard with evenly spaced keys" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/116_keyboard/base2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/116_keyboard/base2.json new file mode 100644 index 00000000..686cc757 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/116_keyboard/base2.json @@ -0,0 +1,22 @@ +{ + "raw_description": "keyboard", + "seen": [ + "gray keyboard", + "plastic gray keyboard", + "medium-sized smooth keyboard", + "smooth plastic gray keyboard", + "keyboard ideal for desktop use", + "keyboard with rectangular layout", + "keyboard with full-sized keycaps", + "keyboard with standard key layout", + "keyboard with numeric pad section", + "keyboard with visible function keys", + "rectangular gray keyboard for input", + "gray rectangular keyboard for typing" + ], + "unseen": [ + "keyboard with flat top surface", + "medium gray keyboard for computers", + "keyboard with keys arranged in rows" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/116_keyboard/base3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/116_keyboard/base3.json new file mode 100644 index 00000000..4f094d5e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/116_keyboard/base3.json @@ -0,0 +1,22 @@ +{ + "raw_description": "keyboard", + "seen": [ + "116-key keyboard", + "colorful keyboard", + "keyboard with purple knob", + "medium-sized RGB keyboard", + "keyboard for typing and gaming", + "black keyboard with colorful lights", + "keyboard with textured plastic keys", + "keyboard with individually lit keys", + "rectangular keyboard with rainbow keys", + "black keyboard with raised colored keys", + "rainbow-lit keyboard with circular knob", + "keyboard featuring metal frame and plastic keys" + ], + "unseen": [ + "keyboard with smooth black frame", + "desk keyboard with purple controls", + "keyboard with matte mixed-color keys" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/117_whiteboard-eraser/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/117_whiteboard-eraser/base0.json new file mode 100644 index 00000000..283c89f8 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/117_whiteboard-eraser/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "whiteboard eraser", + "seen": [ + "whiteboard eraser", + "small rounded block eraser", + "soft felted black side eraser", + "grey top with black felt bottom", + "rounded grey rectangular eraser", + "black and grey whiteboard eraser", + "grey plastic eraser with black pad", + "grey base topped with black soft felt", + "rectangular black eraser for whiteboards", + "light grey rectangular whiteboard eraser", + "compact eraser with black felt underside", + "whiteboard eraser with textured grey grip" + ], + "unseen": [ + "grey eraser", + "eraser with soft black bottom", + "rectangular eraser smooth grey and soft black" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/118_tooth-paste/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/118_tooth-paste/base0.json new file mode 100644 index 00000000..64ac4455 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/118_tooth-paste/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "toothpaste", + "seen": [ + "white toothpaste tube", + "tooth-care paste tube", + "smooth tube for toothpaste", + "toothpaste tube with screw cap", + "hand-sized toothpaste packaging", + "toothpaste tube with green label", + "cylindrical toothpaste container", + "white tube with green shield design", + "toothpaste tube with tapered nozzle", + "plastic tube for holding toothpaste", + "white plastic tube with blue patterns", + "cylindrical white tube for toothpaste" + ], + "unseen": [ + "tube of oral cleaning paste", + "soft plastic toothpaste tube", + "toothpaste tube with smooth texture" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/119_mini-chalkboard/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/119_mini-chalkboard/base0.json new file mode 100644 index 00000000..71ea8bb4 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/119_mini-chalkboard/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "mini-chalkboard", + "seen": [ + "black mini-chalkboard", + "small rectangular chalkboard", + "mini chalkboard for quick notes", + "black surface surrounded by wood", + "small blackboard on wooden holder", + "black writing board with light wood", + "compact blackboard with sturdy base", + "light wooden frame around dark board", + "mini-chalkboard with angled wooden stand", + "rectangular board with light wooden legs", + "small blackboard with angled wooden support", + "desk-friendly chalkboard with polished wooden frame" + ], + "unseen": [ + "tiny chalkboard with stand", + "mini-chalkboard with wooden stand", + "mini-chalkboard for handheld writing" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/120_plant/base0.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/120_plant/base0.json new file mode 100644 index 00000000..7ce186d6 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/objects_description/120_plant/base0.json @@ -0,0 +1,22 @@ +{ + "raw_description": "plant", + "seen": [ + "green plant", + "small plant in white pot", + "small green plant for decoration", + "green leafy plant with white base", + "decorative plant in cylindrical pot", + "soft leaves and smooth white holder", + "compact plant with oval-shaped leaves", + "tabletop plant with bright green leaves", + "bright green leaves with smooth white pot", + "potted plant with white ceramic container", + "tabletop size plant with glossy green leaves", + "plant with long pointed leaves and white base" + ], + "unseen": [ + "green plant with ceramic pot", + "simple green plant in white cylinder pot", + "cylindrical pot holding green leafy plant" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/adjust_bottle.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/adjust_bottle.json new file mode 100644 index 00000000..e19302fd --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/adjust_bottle.json @@ -0,0 +1,69 @@ +{ + "full_description": "Pick up the bottle on the table headup with the correct arm", + "schema": "{A} notifies the bottle, {a} notifies the arm to manipulate the bottle", + "preference": "num of words should not exceed 15", + "seen": [ + "Lift {A} head-up from the table.", + "Pick {A} up with {a} ensuring it stays head-up.", + "Grab {A} from the table and hold it head-up.", + "Utilize {a} to lift {A} while keeping it head-up.", + "Lift {A} ensuring it remains upright.", + "Grab {A} head-up using {a} from the table.", + "Hold {A} head-up after lifting it.", + "Use {a} to pick {A} up and keep it head-up.", + "Pick {A} head-up and hold it steady.", + "Use {a} to grab and lift {A} head-up.", + "Grab {A} from the table with {a}", + "Lift the bottle {A} headup from the table", + "Raise {A} in a head-up position", + "Use {a} to lift {A} head-up", + "Position {A} head-up and lift it", + "Grab {A} with {a} in a head-up way", + "Lift the bottle {A} up from the table", + "Pick {A} head-up using the right arm", + "Grab {A} and lift it into a head-up position", + "Use {a} to pick up {A} in the correct orientation", + "Lift {A} from the table upright", + "Use {a} to hold {A} from the table", + "Identify {A} and grab it with {a}", + "Pick {A} upright from the table", + "Lift {A} using {a} and hold upright", + "Take {A} from the table and keep upright", + "Grab {A} and lift it with {a}", + "Pick up {A} upright from the table", + "Hold {A} upright after lifting with {a}", + "Lift {A} from the table and secure upright", + "Lift {A} from the table with {a}", + "Pick {A} upright from the table", + "Grab {A} and lift it upright", + "Lift {A} head-up from the table", + "Using {a}, pick {A} upright", + "Secure {A} upright with {a}", + "Hold {A} upright from the table", + "Pick {A} and keep it upright", + "Lift {A} upright carefully using {a}", + "Carefully grab {A} head-up", + "Pick up {A} from the table carefully.", + "Use {a} to pick up {A} from the table.", + "Locate {A} and lift it upright with {a}.", + "Raise {A} from the table using the correct arm, {a}.", + "Grab {A} and lift it upward from the table.", + "Use the correct arm to pick up {A}.", + "Lift {A} off the table and hold it upright.", + "Pick up {A} from the table using {a}.", + "Lift {A} from the table without mentioning the arm.", + "Find {A} on the table and raise it using {a}." + ], + "unseen": [ + "Pick up {A} from the table head-up.", + "Use {a} to grab {A} head-up.", + "Use {a} to grab the bottle {A}", + "Pick up {A} using the correct arm", + "Grab {A} from the table with {a}", + "Pick up {A} carefully using {a}", + "Pick up {A} using {a} in an upright position", + "Use {a} to grab {A} upright", + "Lift {A} from the table using {a}.", + "Grab {A} on the table and raise it." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/beat_block_hammer.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/beat_block_hammer.json new file mode 100644 index 00000000..e53b42f8 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/beat_block_hammer.json @@ -0,0 +1,69 @@ +{ + "full_description": "there is a hammer and a block on the table, use the arm to and ", + "schema": "{A} notifies the hammer, {a} notifies the arm to grab the hammer", + "preference": "num of words should not exceed 10", + "seen": [ + "Pick {A} and strike the block.", + "Lift {A} using {a} to hit the block.", + "Take {A} and smash the block.", + "Employ {a} to grab {A} and hit.", + "Hold {A} and pound the block.", + "Utilize {a} to hold {A} and strike.", + "Catch {A} and use it on the block.", + "Grab {A} with {a} and beat the block.", + "Grip {A} firmly and hit the block.", + "Make {a} grab {A} and strike the block.", + "Pick up {A} with {a}, strike the block.", + "Lift {A}, then beat the block.", + "Take {A} using {a}, hit the block.", + "Grab {A}, then strike the block.", + "With {a}, grab {A} and beat the block.", + "Use {A} to hammer the block.", + "With {a}, pick up {A} and strike the block.", + "Grab {A} to beat the block.", + "Pick up {A} using {a}, hammer the block.", + "Lift {A} and hit the block.", + "With {a}, grab {A} and hit the block", + "Pick up {A} and strike the block", + "Grab {A} using {a}, then beat the block", + "Take {A} and hammer the block", + "Grab {A} with {a} and hit the block", + "Pick up {A} and use it on the block", + "With {a}, grab {A} and hammer the block", + "Take {A} and beat the block", + "Grab {A} using {a} and strike the block", + "Lift {A} and hit the block", + "Beat the block after grabbing {A}", + "Grab {A} with {a} and strike block", + "Hold {A} then hit the block", + "Grab {A} using {a} then pound block", + "Pick {A} and smash the block", + "Lift {A} with {a} then strike block", + "Grab {A}, then hit the block", + "Use {a} to grab {A} and beat block", + "Take {A} and strike the block", + "Grab {A} with {a} and hit block", + "Use {a} to grab {A} and beat", + "Grab {A} using {a} and hit the block", + "Take {A} and strike the block", + "Pick up {A} using {a} then beat", + "Use {A} to beat after grabbing with {a}", + "Grab {A} and use it to hit", + "Use {a} to take {A} and strike", + "Pick {A} with {a} and hit the block", + "Take {A} and beat the block", + "Pick {A} from the table and strike" + ], + "unseen": [ + "Grab {A} and beat the block.", + "Use {a} to pick up {A}.", + "Use {a} to grab {A}, beat the block.", + "Grab {A} and hit the block.", + "Grab {A} and beat the block", + "Use {A} to strike the block", + "Grab {A} and hit the block", + "Use {a} to grab {A} then beat block", + "Grab {A} and strike the block", + "Pick {A} up and hit the block" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/blocks_ranking_rgb.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/blocks_ranking_rgb.json new file mode 100644 index 00000000..b6dacd81 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/blocks_ranking_rgb.json @@ -0,0 +1,69 @@ +{ + "full_description": " the red block, green block, and blue block of red, green, and blue from left to right, .", + "schema": "{A} notifies the red block, {B} notifies the green block, {C} notifies the blue block, {a} notifies the arm to grab red block, {b} notifies the arm to grab green block, {c} notifies the arm to grab blue block", + "preference": "num of words should not exceed 30", + "seen": [ + "Set {A}, {B}, and {C} side by side in the order of {A}, {B}, then {C}.", + "Use {a} to grab {A} and position it on the left, then {b} for {B}, and {c} for {C}.", + "Grab {A} using {a}, {B} with {b}, and {C} with {c}, aligning them consecutively in a row.", + "Start with {A}, followed by {B} and {C}, placing them in order left to right.", + "Position {A}, {B}, and {C} from left to right in the specified sequence.", + "Grab {A} with {a}, place it leftmost, then {B} using {b}, followed by {C} with {c}.", + "Begin by placing {A} on the left, then add {B} to the right, and finish with {C}.", + "Use {a} for {A}, then {b} for {B}, and {c} for {C}, arranging them left to right.", + "Arrange {A}, {B}, and {C} in sequence from leftmost to rightmost positions.", + "Pick up {A}, {B}, and {C sequentially and place them in a row from left to right.", + "Position {A} first, followed by {B}, and end with {C} in a row from left to right.", + "Grab {A}, place it on the leftmost spot, add {B} to the middle, and set {C} last.", + "Arrange {A}, {B}, and {C} in a row using {a}, {b}, and {c}, keeping the left-to-right order.", + "Pick up {A}, position it on the left, add {B} next to it, and finally set {C} on the far right.", + "Use {a} to grab {A}, set it on the left, follow with {b} for {B}, and end with {c} for {C}.", + "Arrange {A} at the left edge, {B} to the right of it, and {C} next to {B} forming a row.", + "Pick up {A} using {a}, set it on the left, repeat the placement with {b} for {B}, and {c} for {C}.", + "First, place {A} on the far left, then {B} in the middle, and finish with {C} on the right.", + "Grab {A}, set it on the leftmost side, add {B} to its right, and complete with {C} on the far right.", + "Using {a}, {b}, and {c}, arrange {A}, {B}, and {C} sequentially from left to right in a single row.", + "Place {A} to the left, {B} in the middle, and {C} on the right in a linear fashion.", + "Position {A} first, {B} second, and {C} third in a row from left to right.", + "Using {a}, {b}, and {c}, arrange {A}, {B}, and {C} from left to right in the given order.", + "With {a}, {b}, and {c}, place {A}, {B}, and {C} sequentially from left to right as red, green, blue.", + "Arrange {A}, {B}, and {C} from left to right using {a}, {b}, and {c} in the order red, green, blue.", + "First, grab {A}, then {B}, then {C} with {a}, {b}, and {c}, and place them in order left to right.", + "Position {A}, {B}, and {C} on a surface from left to right in the order red, green, blue.", + "Grab {A}, {B}, and {C}, and place them steadily from left to right in the order red, green, blue.", + "Using {a}, {b}, and {c}, set {A}, {B}, and {C} from left to right in the sequence red, green, blue.", + "Put {A} first, {B} next, and {C} last in a straight line from left to right, using {a}, {b}, and {c}.", + "Grab and place {A} first, followed by {B}, and finish with {C} in a row.", + "Position {A}, {B}, and {C} in a left-to-right sequence, forming a row.", + "First, place {A}, then {B} beside it, and finally {C} next to {B}.", + "Using {a}, {b}, and {c}, align {A}, {B}, and {C} in a row from left to right.", + "Line up the objects in the order {A}, {B}, and {C}, starting from the left.", + "Pick up {A}, {B}, and {C} in sequence and position them in a left-to-right row.", + "Start with {A} on the left, add {B} next to it, and then place {C} on the right.", + "{A}, then {B}, and finally {C} should be placed in a row from left to right.", + "With {a}, {b}, and {c}, arrange {A}, {B}, and {C} sequentially in a horizontal row.", + "Place {A} first, set {B} beside it, and complete the row by positioning {C}.", + "Use {a} to place {A} on the left, then use {b} for {B} beside {A}, and finally {c} for {C} next to {B}.", + "Grab {A}, position it on the left. Next, grab {B} to place beside {A}, then grab {C} and set it to the right of {B}.", + "First, use {a} to grab {A} and place it on the left. Then grab {B} with {b} and position it next to {A}, followed by grabbing {C} with {c} and placing it next to {B}.", + "Start with {A} on the left, followed by {B} next to {A}, and end with {C} on the far right.", + "Pick {A} using {a}, place it far left. Then grab {B} with {b}, position next to {A}. Lastly, grab {C} using {c} and set right of {B}.", + "Position {A} in the first spot, then set {B} to the right of {A}, and finish by placing {C} to the right of {B}.", + "Move {A} with {a} to the far left, {B} with {b} just beside {A}, and finally {C} with {c} next to {B}.", + "Start by grabbing {A}, place it on the leftmost side. Next, position {B} beside {A}, and finally set {C} to the right of {B}.", + "Use {a} for {A} to set it leftmost, continue with {b} to position {B} next to {A}, and finish by placing {C} using {c} to the right of {B}.", + "Place {A} first on the far left, {B} next to it, and {C} last on the far right." + ], + "unseen": [ + "Arrange {A}, {B}, and {C} from left to right in a row.", + "Place {A}, {B}, and {C} sequentially in a row, starting with {A}.", + "Place {A} on the left, then set {B} to its right and {C} next to {B}.", + "Use {a} to arrange {A} first, then use {b} for {B}, and finally use {c} for {C} in a row.", + "Set {A}, {B}, and {C} in a row from left to right in the order red, green, blue.", + "Arrange {A}, {B}, and {C} in a row with {A} on the left, {B} in the middle, and {C} on the right.", + "Set {A} on the left, then {B} in the center, and finally {C} on the right.", + "Arrange {A}, {B}, and {C} side by side, starting with {A} on the left.", + "Pick up {A}, set it on the left. Then grab {B}, position it next to {A}. Finally, place {C} to the right of {B}.", + "Start by placing {A} to the left, followed by {B} next to {A}, and end with {C} on the far right." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/blocks_ranking_size.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/blocks_ranking_size.json new file mode 100644 index 00000000..619b9301 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/blocks_ranking_size.json @@ -0,0 +1,69 @@ +{ + "full_description": "there are three blocks on the table, the color of the blocks is random, move the blocks to the center of the table, and arrange them from largest to smallest, from left to right", + "schema": "{A} notifies the large size block, {B} notifies the medium size block, {C} notifies the small size block, {a} notifies the arm to grab large size block, {b} notifies the arm to grab medium size block, {c} notifies the arm to grab small size block", + "preference": "num of words should not exceed 15. Degree of detail avg 8", + "seen": [ + "Move {A}, {B}, and {C} to the center and align.", + "Pick {A} first, {B} next, and {C} last, and arrange them.", + "Take {A} with {a}, {B} with {b}, and {C} with {c} to the middle.", + "Use {a}, {b}, and {c} to center {A}, {B}, and {C} in order.", + "Center {A}, {B}, {C} with {a}, {b}, and {c} following size order.", + "Shift {A}, {B}, and {C} to the center and align by size.", + "Move {A}, {B}, and {C} to the table center in size order.", + "Relocate {A}, {B}, and {C} to the center and arrange them by size.", + "Pick {A}, {B}, and {C} sequentially and move them to the center.", + "Use {a}, {b}, {c} to place {A}, {B}, and {C} by size at the center.", + "Position {A}, {B}, then {C} at the center in size order.", + "Grab {A}, {B}, {C}, move them, and arrange by size centrally.", + "Shift {A}, {B}, {C} to the center; align by size left to right.", + "Set {A} first, {B} second, and {C} third, ordered at the center.", + "Grab {A}, place it, then {B} and {C}, arranging by size centrally.", + "Position {A}, {B}, and {C} centrally, ordered from largest to smallest.", + "Grab {A}, {B}, {C}, and arrange them in size order at the center.", + "Move {A}, {B}, and {C} to the center, arranging them by size.", + "Place {A}, {B}, {C} centrally and arrange largest to smallest.", + "Grab {A}, {B}, and {C}, set them centrally, ordered by size.", + "Using {a}, place {A} at the table's center-left.", + "Set {B} next to {A} in the center using {b}.", + "Place {C} right of {B} to complete the arrangement.", + "Move the largest block to the center-left of the table.", + "Using {b}, position {B} centrally, next to {A}.", + "Arrange blocks {A}, {B}, and {C} in decreasing size order.", + "Put {A}, {B}, and {C} in the center by size.", + "Use {c} to move {C} next to {B} and complete the lineup.", + "Move all blocks to the center and sort them by size.", + "Using your tools, align {A}, {B}, {C} by size in the center.", + "Arrange {A}, {B}, {C} in size order, largest on the left.", + "Bring {A}, {B}, and {C} to the center, sorting largest to smallest.", + "Transfer {A}, {B}, {C} to the center, aligning from largest to smallest.", + "Use {a}, {b}, and {c} to arrange {A}, {B}, and {C} centrally by size.", + "Grab {A}, {B}, {C} with {a}, {b}, {c} and organize them by descending size.", + "Pick up {A} using {a}, move it to the center, followed by {B} and {C}.", + "Align {A}, {B}, and {C} in the center by size, using {a}, {b}, and {c}.", + "Place {A}, {B}, and {C} at the center and sort them from largest to smallest.", + "Organize {A}, {B}, and {C} in the center of the table by size, largest first.", + "Set {A}, {B}, and {C} in the center area, arranging from largest to smallest.", + "Arrange {A}, {B}, and {C} by size at the center.", + "Use {a}, {b}, and {c} to bring {A}, {B}, {C} to the center.", + "Move {A} with {a}, {B} with {b}, and {C} with {c} to arrange them.", + "Grab {A}, {B}, and {C} and align them by size at the center.", + "Bring {A}, {B}, and {C} to the center using {a}, {b}, and {c}.", + "Shift {A}, {B}, and {C} to the table's center and sort by size.", + "Use {a}, {b}, and {c} to arrange {A}, {B}, {C} in order here.", + "Align {A}, {B}, and {C} by size from left to right at the center.", + "Move {A} with {a}, {B} with {b}, and {C} with {c} to sort them.", + "Bring {A}, {B}, {C} to the table's center and position them left to right." + ], + "unseen": [ + "Place {A}, {B}, and {C} at the table center.", + "Take {A}, {B}, and {C} to the table center.", + "Place {A}, {B}, and {C} in order at the table's center.", + "Move {A}, {B}, and {C} to the center, arrange left to right.", + "Move {A}, {B}, and {C} to the table center.", + "Arrange {A}, {B}, and {C} from largest to smallest.", + "Place {A}, {B}, and {C} at the table center, largest to smallest.", + "Move {A}, {B}, and {C} to the center, ordering largest to smallest.", + "Place {A}, {B}, and {C} in order at the center.", + "Move {A}, {B}, and {C} to the center table." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/click_alarmclock.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/click_alarmclock.json new file mode 100644 index 00000000..d70374d0 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/click_alarmclock.json @@ -0,0 +1,69 @@ +{ + "full_description": "click the alarm clock's center of the top side button on the table", + "schema": "{A} notifies the alarm clock, {a} notifies the arm to click the alarm clock", + "preference": "num of words should not exceed 10", + "seen": [ + "{a} clicks the center top button on {A}", + "Locate and press the top button on {A}", + "Activate {A} by pressing the top button", + "Press the center top button on {A} with {a}", + "Use {a} to click the center button on {A}", + "Click the top center button of {A}", + "Press the button on {A}'s top side", + "{a} presses the center top button on {A}", + "Ensure {a} clicks the top center button of {A}", + "{a} activates {A} by clicking the top button", + "Tap the center button on {A}.", + "Click {A}'s top center button using {a}.", + "Touch {A}'s button on the top side.", + "Point {a} to press the button of {A}.", + "Click the button centered on {A}.", + "Use {a} to tap {A}'s central button.", + "Press the top button at {A}'s center.", + "Guide {a} to press the centered button on {A}.", + "Tap the button at {A}'s top center.", + "Direct {a} to click the button found on {A}.", + "Click the center of {A}'s top", + "Use {a} to press {A} button", + "Use {a} to click the top of {A}", + "Tap {A}'s top button with {a}", + "Push the top button on {A}", + "Activate {A} by pressing its top", + "Use {a} to activate {A}'s button", + "Press the center top button of {A}", + "Push the center area of {A}'s top", + "Use {a} to press the center top of {A}", + "Use {a} to press {A}'s top button", + "Press the top middle button of {A}", + "Touch {A}'s center top button with {a}", + "Locate and press {A}'s top center button", + "Activate {A} by clicking its top button", + "Push the top button of {A} using {a}", + "Click the middle button on {A}'s top side", + "Use {a} to tap {A}'s top button center", + "Hit the center of {A}'s top button", + "Press {A}'s top button using {a}", + "Use {a} to press {A}'s top center button.", + "Push the button on {A}'s top center.", + "Press the center button on {A}'s top.", + "Use {a} to tap {A}'s top center button.", + "Click the button centered on {A}'s top.", + "Press the top button in {A}'s center.", + "Use {a} to click the top center button on {A}.", + "Push the central button on {A}'s top side.", + "Use {a} to press the central button on {A}.", + "Click the center button on {A}'s top side." + ], + "unseen": [ + "Click the center top button of {A}", + "Press the top button on {A}", + "Press the top center of {A}.", + "Use {a} to click {A}'s top center button.", + "Click the top button of {A}", + "Press {A}'s center top button", + "Click the top button of {A}", + "Tap the center top button of {A}", + "Press {A}'s top center button.", + "Click the central top button on {A}." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/click_bell.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/click_bell.json new file mode 100644 index 00000000..9dbc4d16 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/click_bell.json @@ -0,0 +1,69 @@ +{ + "full_description": "click the on the table", + "schema": "{A} notifies the bell, {a} notifies the arm to click the bell", + "preference": "num of words should not exceed 10", + "seen": [ + "Press using {a} on the table", + "Instruct {a} to press ", + "Push on the table", + "Click {A}'s using {a}", + "Make {a} press ", + "Press the directly", + "Direct {a} to click ", + "Push {A}'s on the table", + "Click using {a}", + "Press placed on the table", + "Press the center top of {A}.", + "Command {a} to press {A}'s top.", + "Click at the bell's top center.", + "Direct {a} to touch {A}'s top.", + "Press down on the bell's top.", + "Guide {a} to click the bell's top.", + "Click the designated center of {A}.", + "Request {a} to press the bell's top.", + "Press the specified top area of {A}.", + "Ask {a} to interact with {A}'s top.", + "Press the center of {A} using {a}.", + "Click the bell's center on the table.", + "Tap {A}'s top center with {a}.", + "Tap the top center of {A}.", + "Press {A}'s top center on the table.", + "Click using {a} on {A}'s center.", + "Push the center of {A} using {a}.", + "Push the bell's center on the table.", + "Press down {A}'s top center gently.", + "Press down the top of {A} using {a}.", + "Click the top center of {A} on table.", + "Direct {a} to click the top of {A}.", + "Pinpoint {A} and click its top center.", + "Have {a} click at {A}'s top center.", + "Press the top center of {A} on table.", + "Make {a} interact with {A}'s top center.", + "Click {A} at its top center on table.", + "Guide {a} to click {A}'s top center.", + "Locate {A} and click its top center.", + "Use {a} to press {A}'s top section.", + "Engage the top center of the bell.", + "Click {A}'s top center using {a}.", + "Press the bell's top center on the table.", + "Tap {A}'s top center with {a}.", + "Touch the bell at its top center.", + "Use {a} to touch {A}'s top center.", + "Engage the bell's top center gently.", + "Activate {A} by pressing its top center.", + "Press {A}'s top center with {a} firmly.", + "Tap the bell's top center on the table." + ], + "unseen": [ + "Click the on the table", + "Tap the placed on the table", + "Click the top center of {A}.", + "Direct {a} to click {A}'s top.", + "Click {A}'s top center on the table.", + "Use {a} to press {A}'s top center.", + "Find {A} and click its top center.", + "Use {a} to press {A}'s top center.", + "Click the bell at its top center.", + "Use {a} to press {A}'s top center." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/dump_bin_bigbin.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/dump_bin_bigbin.json new file mode 100644 index 00000000..d9527288 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/dump_bin_bigbin.json @@ -0,0 +1,69 @@ +{ + "full_description": "Grab the small bin and pour the balls into the big bin", + "schema": "{A} notifies the small bin", + "preference": "num of words should not exceed 10. Degree of detail avg 5", + "seen": [ + "Grab {A} and empty it into the bin.", + "Take {A} and pour the balls out.", + "Hold {A} and dump its balls.", + "Use the arm to move {A} and pour.", + "Direct the arm to grab {A} and tilt.", + "Control the arm to raise {A} and pour.", + "Use the arm to empty {A} into the bin.", + "Move the arm, take {A}, and pour.", + "Grab and tilt {A} to empty the balls.", + "Raise {A} to dump its contents out.", + "Lift {A} and pour balls into the bin.", + "Use the arm to grab {A} and pour.", + "Take {A} and empty balls into the bin.", + "Grab {A}, then pour balls into the bin.", + "Pick up {A} using the arm, pour balls.", + "Lift {A} using the arm to pour balls.", + "Take hold of {A} and dump the balls.", + "Use the arm to lift {A} and empty it.", + "Grab {A} and transfer balls to the bin.", + "Lift {A} and pour its contents into bin.", + "Pick up {A}, empty into the big bin", + "Lift {A}, pour contents into big bin", + "Use arm to pick {A}, pour in bin", + "Grab {A} with arm and pour contents", + "Hold {A}, dump balls in big bin", + "Use arm to grab {A}, dump contents", + "Pick {A} up, pour all balls inside bin", + "Take {A} using arm, pour contents inside", + "Lift {A}, pour the balls into big bin", + "Hold {A} and pour all balls into bin", + "Hold {A}, then pour contents.", + "Grab {A} and pour it out.", + "Lift {A}, pour into the big bin.", + "Pour balls from {A} into the bin.", + "Take {A} and empty its contents.", + "Grab {A}, pour its contents away.", + "Lift {A} and pour contents down.", + "Use {A} to pour the balls out.", + "Pick up {A} and pour into bin.", + "Pick {A}, pour its contents out.", + "Take {A} and pour contents.", + "Lift {A}, then pour the balls.", + "Grab {A} and transfer balls.", + "Pick up {A} and pour carefully.", + "Take hold of {A}, pour contents.", + "Secure {A} and pour the balls.", + "Grasp {A} and empty it out.", + "Hold {A} and pour the balls.", + "Lift up {A}, then pour balls.", + "Take {A} and pour the balls." + ], + "unseen": [ + "Pick up {A} and pour it.", + "Lift {A} and transfer the contents.", + "Grab {A} and pour the balls.", + "Pick up {A}, pour balls into the bin.", + "Grab {A} and pour into big bin", + "Take {A} and empty into big bin", + "Take {A} and empty it.", + "Pour {A} into the big bin.", + "Grab {A} and pour balls.", + "Lift {A} and empty it." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/grab_roller.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/grab_roller.json new file mode 100644 index 00000000..a1a11d52 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/grab_roller.json @@ -0,0 +1,69 @@ +{ + "full_description": "use both arms to grab the roller on the table", + "schema": "{A} notifies the roller. 'arm' use literal here", + "preference": "num of words should not exceed 10.", + "seen": [ + "Take hold of {A} using your arms.", + "Firmly grip {A} on the table now.", + "Grab {A} directly from the table.", + "Take both arms to grasp {A}.", + "Hold {A} on the table with hands.", + "Reach for {A} and grab it firmly.", + "Securely grab {A} using both arms.", + "Use both arms to grip {A} tightly.", + "Grasp {A} firmly from the table.", + "Lift {A} off the table with arms.", + "Secure {A} on the table using arms.", + "Take hold of {A} with both arms.", + "Grab the roller on the table.", + "Hold {A} firmly from the table.", + "Lift {A} from the table carefully.", + "Take {A} directly from the table.", + "Grasp {A} on the table with arms.", + "Use arms to firmly grab {A}.", + "Both arms should grab {A} now.", + "Pick up {A} from the table directly.", + "Get hold of {A} using your arms", + "Secure {A} from the table using arms", + "Grab the roller placed on the table", + "Lift {A} off the table with both arms", + "Reach for {A} and hold it firmly", + "Lift {A} from its place on the table", + "Pick up {A} using both arms equally", + "Pick up the roller using any method", + "Hold {A} with both arms to pick it up", + "Reach out to grab {A} from the table", + "Grab the roller on the table.", + "Secure {A} with both arms.", + "Grab {A} placed on the table.", + "Lift {A} with your arms.", + "Pick up {A} using both arms.", + "Grab roller using both hands.", + "Grasp {A} firmly with arms.", + "Hold {A} from the table.", + "Take hold of {A} with arms.", + "Lift {A} from the table.", + "Grip {A} firmly with arms.", + "Use both arms to grab {A}.", + "Take hold of {A}.", + "Secure {A} using your arms.", + "Pick up {A} from the table.", + "Bring both arms to grab {A}.", + "Place hands on {A} and lift.", + "Use arms to hold {A} tightly.", + "Grasp {A} on the table.", + "Firmly grab {A} using arms." + ], + "unseen": [ + "Grab {A} on the table with arms.", + "Use both arms to grab {A}.", + "Grab {A} on the table with arms.", + "Use both arms to grab {A}.", + "Grab {A} on the table using arms", + "Reach and grab {A} with both arms", + "Grab {A} with both arms.", + "Use arms to grab {A}.", + "Hold {A} with both arms.", + "Grab {A} on the table." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/handover_block.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/handover_block.json new file mode 100644 index 00000000..83a2d1ca --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/handover_block.json @@ -0,0 +1,69 @@ +{ + "full_description": "use the left arm to grasp the red block on the table, handover it to the right arm and place it on the blue pad", + "schema": "", + "preference": "num of words should not exceed 15", + "seen": [ + "Pass the red block from the left to the right.", + "Place the red block onto the blue pad using the right arm.", + "Pick the red block with the left hand and pass it.", + "Hand over the red block to the right arm.", + "Set the red block on the blue pad gently.", + "Take the red block using the left arm and hand it over.", + "Grip the red block with the left arm and transfer it.", + "Move the red block from the left hand to the pad.", + "Grab the red block, transfer it, and set it on blue.", + "Lift the red block, hand it over, and place it properly.", + "Use left arm to grab the block and hand it to the right.", + "Take the red block, pass it to the right arm, and drop it.", + "Pick the red block with the left arm and transfer to the right.", + "Hold the red block with the left arm and place it on the pad.", + "Grab the red block, hand it to the right arm, then set it down.", + "Lift the block with the left arm, pass it to the right arm.", + "Use left arm to grab the red block and place it on the pad.", + "Take the red block, pass it to the right arm, then place it.", + "Grab red block, pass to the right arm, then drop on blue pad.", + "Hold the red block, transfer to the right arm, then set it down.", + "Pick up the red block using your left arm, pass it to the right arm, and place it on the blue pad.", + "With the left arm grab the red block, hand it to the right arm, then position it onto the blue pad.", + "Lift the red block off the table using the left arm, give it to the right hand, and put it on the blue pad.", + "Grasp the block in your left arm, pass it into the right, and place it onto the blue pad.", + "Grasp the red block with your left arm, transfer it into the right, and place it on the blue pad.", + "Take the red block from the table using your left arm, hand it off to the right one, and rest it on the blue pad.", + "The left arm should grab the red block, hand it to the right, and put it on the blue pad.", + "Pick the red block from the table, switch it from the left arm to the right, and place it down on the blue pad.", + "Lift the red block with the left arm, move it to the right, and place it onto the blue pad.", + "With the left arm, grab the red block, pass it to the right, and set it on the blue pad.", + "Move the red block using the left arm.", + "Lift the red block with your left arm.", + "Hand the red block to the right arm.", + "Grab the red block and transfer to the right.", + "Use the left arm to grab the red block.", + "Place the red block on the blue pad.", + "Pick up and handover the red block.", + "Take the red block and move it to the pad.", + "Use your arms to shift the red block.", + "Lift, switch arms, and place the red block.", + "Use the left arm to grab the red block, hand it to the right arm, and place it on the blue pad.", + "Lift the red block with the left arm, pass it to the right arm, and place it on the blue pad.", + "Grab the red block on the table with the left arm, switch to the right arm, and drop it on the blue pad.", + "Pick the red block up with the left arm, hand it to the right arm, and set it on the blue pad.", + "Lift the red block from the table with the left arm, pass it to the right arm, and place it on the blue pad.", + "Use the left arm to grasp the red block, transfer it to the right arm, and set it down on the blue pad.", + "Pick up the red block using the left arm, give it to the right arm, and place it on the blue pad.", + "Grab the red block with the left arm, pass it to the right hand, and drop it on the blue pad.", + "Use the left arm to pick up the red block and transfer it to the right arm before placing it on the blue pad.", + "Lift the red block with the left arm, shift it to the right arm, and set it on the blue pad." + ], + "unseen": [ + "Grab the red block with the left arm.", + "Use the left arm to pick up the red block.", + "Grab the red block using the left arm.", + "Pass the red block to the right arm and set it down.", + "Grab the red block with the left arm, switch to the right, and place it on the blue pad.", + "Use the left arm to grab the red block, transfer it to the right hand, and set it onto the blue pad.", + "Grab the red block with the left arm.", + "Pick up the red block and switch arms.", + "Grab the red block with the left arm, switch to the right arm, and set it on the blue pad.", + "Pick up the red block with the left arm, transfer it to the right, and place it on the blue pad." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/handover_mic.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/handover_mic.json new file mode 100644 index 00000000..ef125af1 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/handover_mic.json @@ -0,0 +1,69 @@ +{ + "full_description": "Use one arm to grasp the microphone on the table and handover it to the other arm", + "schema": "{A} notifies the microphone, {a} notifies the arm to grab the microphone, {b} notifies the arm to hand over to", + "preference": "num of words should not exceed 15", + "seen": [ + "Pick {A} and transfer it to the other arm.", + "Hold {A} and pass it to the other hand.", + "Grasp {A}, then give it to the other arm.", + "Lift {A} and pass it across.", + "Secure {A} using one arm and transfer it.", + "Pick up {A} and hand it to the other side.", + "Grab {A} and give it to the opposite arm.", + "Take {A} and move it to the other hand.", + "Hold {A} firmly and pass it to the other arm.", + "Lift {A} and deliver it to the other side.", + "Use {a} to grab {A} and transfer it to {b}.", + "Lift {A} and hand it over to the other arm.", + "Grasp {A} and pass it across.", + "Take {A} and move it to another hand.", + "Hold {A} and deliver it to another side.", + "Lift {A} and hand it to someone else.", + "Use one hand to grab {A} and pass it.", + "Grasp {A} and switch it to another hand.", + "Secure {A} from the table and transfer it.", + "Take hold of {A} and pass it to {b}.", + "Use {a} to hold {A}, then give it to {b}.", + "Hold {A} securely and shift it to another arm.", + "Lift {A} using {a} and pass it to {b}.", + "Pick {A} from the surface and switch hands.", + "Hold {A} with {a} and give it to {b}.", + "Grasp {A} and shift it to the opposite hand.", + "Take {A} using {a} and transfer it to {b}.", + "Lift {A} and hand it over to the other side.", + "Grab {A} using {a} and pass it over to {b}.", + "Reach for {A} and move it to the other hand.", + "Hold {A} with one hand and transfer it", + "Take {A} and give it to the other {b}", + "Grip {A} and pass it to the other side", + "Use one {a} to grab {A} and give it away", + "Lift {A} and place it in the other {b}", + "Seize {A} and offer it to the other arm", + "Take {A} and pass it to another hand", + "Pass {A} from one side to the other {b}", + "Pick up {A} and move it to the opposite side", + "Grab {A} and transfer it to another hand", + "Use one arm to pick up {A} and give it to the other.", + "Pick up {A} and transfer it to the opposite side.", + "Hold {A} and shift it to the other arm.", + "Lift {A}, then pass it across without delay.", + "Grab {A} and smoothly give it to the other arm.", + "Take {A}, shift it, and release it to the other side.", + "Pick up {A}, pass it to the other arm, and release.", + "Lift {A} and hand it to the other side easily.", + "Grasp {A}, transfer it, then let go of it smoothly.", + "Take {A}, pass it, and release it to complete the task." + ], + "unseen": [ + "Grab {A} from the table and pass it over.", + "Use one arm to hold {A} and hand it over.", + "Grab {A} from the table and hand it to {b}.", + "Pick up {A} and pass it to {b}.", + "Pick up {A} and transfer it to the other hand.", + "Grab {A} from the table and pass it across.", + "Grab {A} and pass it to another {b}", + "Pick up {A} and hand it over", + "Grab {A} and pass it to the other arm.", + "Take hold of {A} and hand it over." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/hanging_mug.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/hanging_mug.json new file mode 100644 index 00000000..fcbba5da --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/hanging_mug.json @@ -0,0 +1,69 @@ +{ + "full_description": "Use left arm to pick the mug on the table, rotate the mug and put the mug down in the middle of the table, use the right arm to pick the mug and hang it onto the rack.", + "schema": "{A} notifies the mug, {B} notifies the rack", + "preference": "num of words should not exceed 15", + "seen": [ + "Use the left arm to grab {A}, rotate it, set it down, then hang {A} onto {B}.", + "Lift {A}, turn it, place it back, and move it onto {B} with the right arm.", + "Take {A}, rotate it, place it in the center, then hang it on {B}.", + "Grab {A}, rotate it, place it in the middle, and hang it on {B}.", + "Use your left arm to pick {A}, rotate it, set it down, and hang it onto {B}.", + "Use the right arm to lift {A}, rotate it, and attach it to {B} after setting it down.", + "Pick {A}, turn it, put it in the center, and transfer it to {B}.", + "Lift {A} from the table, rotate it, place it in the middle, and hang it on {B}.", + "Use the left arm to grab {A}, flip it, set it down, and then attach it to {B}.", + "Take {A}, rotate it, place it back on the table, and hang it on {B} afterward.", + "Using the left arm, pick {A}, turn it, place it down, and hang it on {B}.", + "Pick {A} up, twist it, place it back, then hang it onto {B}.", + "With the left arm, grab {A}, spin it, place it down, then use the right arm to hang it on {B}.", + "Take {A}, rotate it, set it in the middle, then hang it on {B}.", + "Using your left arm, lift {A}, rotate it, place it down, then your right arm to hang it onto {B}.", + "Grab {A}, turn it around, put it back, then attach it to {B}.", + "Lift {A}, twist it, set it back, and secure it onto {B}.", + "Use the left arm to grab {A}, rotate it, place it down, and hang it onto {B} with the right arm.", + "Take {A} from the table, spin it, place it in the center, then hang it on {B}.", + "Pick {A} using your left arm, turn it, set it on the table, and hang it on {B} with the right arm.", + "Grab {A}, rotate it, place it on the table, and hook it onto {B}.", + "Use the left arm to pick {A}, turn it, place it, then hang on {B}.", + "Pick {A} from the table using one hand, rotate, and hang it on {B}.", + "Grab {A} from the table, rotate it, place it, and hang it onto {B}.", + "Use your left arm to grab {A}, rotate it, set it down, then hang on {B}.", + "Take {A} from the table, turn it, set it down, and attach it to {B}.", + "With the left arm, pick {A}, rotate it, place it, then hang it onto {B}.", + "Lift {A}, turn it around, set it in the middle, and place it on {B}.", + "Use the left arm to lift {A}, rotate it, put it down, then hook onto {B}.", + "Pick {A} from the table, rotate it, place it, and then hang it onto {B}.", + "Pick {A}, turn it, and leave it in the table's center.", + "Lift {A} with one arm, rotate, and drop it on the table.", + "Pick {A}, rotate it, and place it in the table middle.", + "Use one arm to grab {A}, rotate, and place it down.", + "Lift {A}, rotate, and center it on the table.", + "Take {A} with one arm, turn it, and set it in the center.", + "Grab {A}, twist it, then place it on the table's center.", + "Use one arm to move {A}, rotate it, and position it on the rack.", + "Lift {A}, give it a turn, and hang it onto {B}.", + "Take {A} with one arm, rotate it, and hang it onto {B}.", + "Use the left arm to grab {A}, rotate it, place it in the middle, then use the right arm to hang it onto {B}.", + "Lift {A} from the table, turn it, put it down in the middle, then hang it onto {B}.", + "Take {A}, rotate it, set it on the table's center, then hang it onto {B}.", + "Use your left arm to grab {A}, rotate it, place it in the middle, then use the right arm to hang it onto {B}.", + "Pick up {A}, turn it, place it centrally, then hang {A} onto {B}.", + "With the left arm, grab {A} from the table, rotate it, place it centrally, and with the right arm, hang it onto {B}.", + "Grab {A}, rotate it, place it in the middle, and hang it onto {B}.", + "Use one arm to grab {A}, turn it, set it centrally, and use the other to hang it onto {B}.", + "Lift {A}, rotate it, put it down in the table's center, then hang it onto {B}.", + "Take {A} from the table, rotate it, place it in the center, then hang it onto {B}." + ], + "unseen": [ + "Grab {A} from the table, rotate it, and set it in the center. Then hang {A} on {B}.", + "Pick up {A}, rotate it, place it on the table, and hang it on {B}.", + "Grab {A}, turn it, set it on the table, then hang it on {B}.", + "Lift {A}, rotate it, put it down, then attach it to {B}.", + "Pick {A} with the left arm, rotate, place it, then hang it on {B}.", + "Lift {A} from the table, spin it, set it down, and hang it on {B}.", + "Grab {A} from the table, rotate, and set it down.", + "Use one arm for {A}, rotate, and place it on the table.", + "Grab {A} on the table, rotate it, set it down in the middle, then hang it onto {B}.", + "Pick {A} from the table, rotate it, place it in the middle, and hang it onto {B}." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/lift_pot.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/lift_pot.json new file mode 100644 index 00000000..66c35b53 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/lift_pot.json @@ -0,0 +1,69 @@ +{ + "full_description": "use BOTH!!! arms to lift the pot", + "schema": "{A} notifies the pot. Arm comes as literal here.", + "preference": "num of words should not exceed 6!!!!!. Degree of detail avg is 2.Avoid using adjectives!!", + "seen": [ + "Hold {A} firmly, then lift.", + "Use both arms to raise {A}.", + "Secure {A} and lift upward.", + "Place hands on {A}, then lift.", + "Grasp {A} and elevate together.", + "Lift {A} using both arms now.", + "Engage arms to grip and lift {A}.", + "With arms, raise {A} upward slowly.", + "Hold {A} firmly and move upward.", + "Lift {A} carefully using both arms.", + "Use both arms to raise {A}", + "Grab {A} and lift it upwards", + "Pick up {A} with careful lifting", + "Secure {A} and lift it up", + "Raise {A} steadily using arms", + "Lift {A} upward with both arms", + "Take hold of {A} and lift up", + "Support {A} and raise it upward", + "Lift {A} up using your arms", + "Raise {A} upward with both hands", + "Raise {A} using both arms", + "Bring {A} up together", + "Hold {A} with both arms", + "Lift {A} up together", + "Raise {A} evenly with arms", + "Bring {A} upwards together", + "Grip {A} firmly and lift", + "Hold and raise {A} together", + "Lift {A} steadily using arms", + "Raise and hold {A} together", + "Hold {A} firmly with arms", + "Securely lift {A} together", + "Raise {A} with strong support", + "Carry {A} securely using arms", + "Grab {A} and lift together", + "Both arms lift {A} upright", + "Lift {A} carefully using arms", + "Hold and raise {A} together", + "Lift {A} steadily with support", + "Raise {A} securely with arms", + "Raise {A} together using arms", + "Grab {A} and lift it up", + "Hold {A} and lift upward", + "Lift {A} upwards with care", + "Grab {A} using both arms", + "Use arms to lift {A} upward", + "Pick up {A} with both arms", + "Hold {A} firmly and lift it", + "Lift {A} upward and hold it", + "Raise {A} together with arms" + ], + "unseen": [ + "Grab {A} with both arms.", + "Lift {A} upward using arms.", + "Lift {A} using both arms", + "Hold {A} firmly and lift it", + "Lift {A} with both arms", + "Together lift {A} up", + "Use both arms for {A}", + "Lift {A} using both arms", + "Lift {A} with both arms", + "Use both arms to lift {A}" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/move_can_pot.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/move_can_pot.json new file mode 100644 index 00000000..b81023e6 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/move_can_pot.json @@ -0,0 +1,69 @@ +{ + "full_description": "there is a can and a pot on the table, use one arm to and ", + "schema": "{A} notifies the pot, {B} notifies the can, {a} notifies the arm to grab the can", + "preference": "num of words should not exceed 10", + "seen": [ + "Use {a} to grab {B} and move it next to {A}", + "Pick {B} up with {a} then place near {A}", + "Move {B} from its spot to near {A}", + "Lift {B} using {a} and drop it beside {A}", + "Grab {B}, shift it, and place it close to {A}", + "Take {B} with {a}, bring it, and set next to {A}", + "Pick {B} up and carefully position it beside {A}", + "Using {a}, lift {B} and place it by {A}", + "Pick {B} with {a} and relocate it near {A}", + "Lift {B} and move it near {A}", + "Pick {B} up and move it next to {A}", + "Grab {B} with {a} and set it near {A}", + "Place {B} beside {A} after picking it up", + "Use {a} to lift {B}, then move it next to {A}", + "Move {B} beside {A} after lifting it", + "Grab {B} with {a} and position it beside {A}", + "Lift {B}, then place it next to {A}", + "Use {a} to grab {B} and move it beside {A}", + "Set {B} near {A} after picking it up", + "Use {a} to lift {B} and place it near {A}", + "Use {a} to take {B} to {A}", + "Lift {B} and place it next to {A}", + "Use {a} to move {B} beside {A}", + "Pick up {B} with {a} and set by {A}", + "Grab {B}, move it, and place by {A}", + "Take {B} to {A} using {a}", + "Set {B} right next to {A}", + "With {a}, grab {B} and move to {A}", + "Lift {B} and set it beside {A}", + "Move {B} to {A} with {a}", + "Lift {B} and set it next to {A}", + "Use {a} to grab {B} and transfer it near {A}", + "Pick up {B} and put it beside {A}", + "Lift {B} using {a} and position it by {A}", + "Move {B} to be next to {A}", + "Use {a} to pick {B} up and set it beside {A}", + "Place {B} next to {A}", + "Grab {B} with {a} and move it close to {A}", + "Bring {B} over and set it near {A}", + "Use {a} to lift {B} and place it next to {A}", + "Lift {B} and set it next to {A}", + "Pick up {B} using {a}, transfer it beside {A}", + "Grab {B}, move it to {A}'s side", + "Take {B} with {a}, place it near {A}", + "Pick {B} and position it next to {A}", + "Use {a} to grab {B}, move it beside {A}", + "Lift {B}, place it by {A}", + "Take {B} using {a}, set it next to {A}", + "Grab {B} and move it close to {A}", + "Use {a} to pick {B}, position it near {A}" + ], + "unseen": [ + "Pick up {B} and move it near {A}", + "Grab {B} and set it beside {A}", + "Lift {B} and set it beside {A}", + "Use {a} to grab {B} and place it by {A}", + "Pick up {B} and set it beside {A}", + "Grab {B} and move it near {A}", + "Grab {B} and place it near {A}", + "Use {a} to pick up {B} and move it beside {A}", + "Grab {B} and place it beside {A}", + "Use {a} to pick up {B}, move it near {A}" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/move_pillbottle_pad.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/move_pillbottle_pad.json new file mode 100644 index 00000000..cfd1bef5 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/move_pillbottle_pad.json @@ -0,0 +1,69 @@ +{ + "full_description": "Use one arm to pick the pillbottle and place it onto the pad.", + "schema": "{A} notifies the pillbottle, {a} notifies the arm to move the pillbottle", + "preference": "num of words should not exceed 10", + "seen": [ + "Move {A} with one arm to the pad", + "Lift {A} with {a} and drop on pad", + "Pick {A} and place it on the pad", + "Use {a} to move {A} to the pad", + "Grab {A} and drop it onto the pad", + "Lift {A} with {a} and set it down", + "Take {A} and put it onto the pad", + "Use {a} to set {A} onto the pad", + "Move {A} and position it on the pad", + "Grab and transfer {A} with {a} to pad", + "Use {a} to lift {A} and place it", + "Move {A} using {a} and drop it on pad", + "Place {A} on pad after lifting with {a}", + "Lift {A} and carefully place it on pad", + "Pick {A} with {a} and position it onto pad", + "Grab {A} and drop it on the pad", + "Use {a} to pick {A}, then place it on pad", + "Pick {A} up and set it onto the pad", + "Set {A} on pad after lifting it with {a}", + "Move {A} to the pad and let it rest there", + "Grab {A} and place it on the pad", + "Use {a} to grab {A} and place it", + "Pick {A} up and drop it onto the pad", + "Move {A} onto the pad using {a}", + "Lift {A} and set it on the pad", + "Use {a} to lift {A} and position it", + "Take {A} and place it onto the pad", + "Use {a} to take {A} and move it", + "Pick {A} using {a} and set it on pad", + "Move {A} to the pad and leave it there", + "Grab {A} with {a} and move it to the pad.", + "Place {A} onto the pad after picking it.", + "Lift {A} using {a} and position it on the pad.", + "Move {A} to the pad and release it there.", + "Pick {A} with {a} and drop it onto the pad.", + "Set {A} down on the pad after picking it.", + "Use {a} to grab {A} and put it on the pad.", + "Place {A} on the pad after lifting it up.", + "Grab {A} and carefully set it onto the pad.", + "Use {a} to pick and place {A} onto the pad.", + "Move {A} and drop it on the pad", + "Pick {A} with {a} and place it down", + "Grab {A} and position it on the pad", + "Use {a} to move {A} to the pad", + "Lift {A} and gently lower onto the pad", + "Control {a} to pick {A} and set it down", + "Grab {A} carefully and drop it on pad", + "Use {a} to place {A} on the pad", + "Hold {A} and position it on the pad", + "Direct {a} to grab {A} and drop it there" + ], + "unseen": [ + "Pick {A} and set it on the pad", + "Use {a} to grab {A} and place it", + "Grab {A} and set it on the pad", + "Pick {A} and put it onto the pad", + "Pick {A} and set it on the pad", + "Use {a} to move {A} onto the pad", + "Use {a} to pick {A} and place it.", + "Pick {A} and set it onto the pad.", + "Pick {A} and set it on the pad", + "Use {a} to grab {A} and place it" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/move_playingcard_away.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/move_playingcard_away.json new file mode 100644 index 00000000..12cecb19 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/move_playingcard_away.json @@ -0,0 +1,69 @@ +{ + "full_description": "use the arm to and .For example, if the playing card is on the outward side of the table, you should move it further outward side of the table.", + "schema": "{A} notifies the playing card, {a} notifies the arm to grab the playing card", + "preference": "num of words should not exceed 10", + "seen": [ + "Move {A} further outward after picking it up.", + "Pick up {A} using {a} and shift it outward.", + "Lift {A} and relocate it off the table.", + "With {a}, pick up {A} and slide it outward.", + "Grab {A} and move it further outward.", + "Pick up {A} with {a} and place it outward.", + "Lift {A} and set it away from the table.", + "Use {a} to grab {A} and move it aside.", + "Pick up {A} and shift it further outward.", + "Using {a}, lift {A} and move it outward.", + "Lift {A} and push it off the table.", + "Pick up {A} and move it away.", + "Move {A} outward from the table.", + "Shift {A} off the table to the side.", + "Take {A} and place it further outward.", + "Slide {A} off the table outward.", + "Grab {A} and move it further away.", + "Lift {A} and shift it outward.", + "Place {A} away from the table outward.", + "Move {A} off the table outward.", + "Use {a} to grab {A} and move it away.", + "Move {A} farther from the table using {a}.", + "Grab {A} and take it away from the table.", + "Pick up {A} and move it farther outward.", + "Use {a} to pick {A} and shift it outward.", + "Take {A} and move it away from the table.", + "Lift {A} and place it farther away with {a}.", + "Shift {A} outward by grabbing it with {a}.", + "Pick {A} and move it outward from the table.", + "Grab {A} and shift it farther away from the table.", + "Move {A} away from its position on the table.", + "Grab {A} from the table and shift it outward.", + "Lift {A} using {a} and transfer it outward.", + "Relocate {A} farther outward using {a}.", + "Pick up {A} and place it further away.", + "Shift {A} outward after grabbing it.", + "Use {a} to pick up {A} and move it.", + "Pick {A} and slide it further outward.", + "Lift {A} from the table and move it outward.", + "Grab {A} using {a} and move it outward.", + "Lift {A} and slide it off the table.", + "Grab {A} with {a} and move it outward.", + "Take {A} and push it further away.", + "Use {a} to pick up {A} and push it outward.", + "Grab {A} and set it beyond the table.", + "Lift {A} with {a} and move it outward.", + "Pick {A} from the table and shift it outward.", + "Use {a} to pick {A} and move it outward.", + "Take {A} and place it beyond the table.", + "Grab {A} using {a} and push it outward." + ], + "unseen": [ + "Use {a} to grab {A} and move it outward.", + "Grab {A} and move it off the table.", + "Grab {A} and move it outward.", + "Pick up {A} and shift it outward.", + "Pick up {A} and move it outward.", + "Lift {A} from the table and shift it.", + "Pick up {A} and move it outward.", + "Use {a} to grab {A} and relocate it.", + "Pick up {A} and move it outward.", + "Use {a} to grab {A} and relocate it." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/move_stapler_pad.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/move_stapler_pad.json new file mode 100644 index 00000000..28403360 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/move_stapler_pad.json @@ -0,0 +1,69 @@ +{ + "full_description": "use appropriate arm to move the stapler to a colored mat", + "schema": "{A} notifies the stapler, {B} notifies the color of the mat(YOU SHOULD SAY {B} mat, or {B} colored mat), {a} notifies the arm to grab the stapler", + "preference": "num of words should not exceed 10", + "seen": [ + "Grab {A} and drop it on {B} mat.", + "{a} moves {A} to the {B} mat.", + "Set {A} onto the {B} colored mat.", + "{a} places {A} on the {B} mat.", + "Drop {A} onto the {B} mat.", + "Stick {A} onto the {B} colored mat.", + "{a} grabs {A} and sets it on {B} mat.", + "Slide {A} to the {B} colored mat.", + "{a} transfers {A} to the {B} mat.", + "Stick {A} on the {B} mat.", + "Grab {A} using {a} and set it on {B} mat.", + "Move {A} to the {B} mat.", + "Transfer {A} to the {B} colored mat.", + "Use {a} and place {A} on {B} mat.", + "Set {A} on the {B} mat with {a}.", + "Position {A} on the {B} mat.", + "Pick {A} using {a} and move it to {B} mat.", + "Place {A} onto the {B} colored mat.", + "Relocate {A} to the {B} mat.", + "Grab {A} with {a} and drop it on {B} mat.", + "Grab {A}, place it on the {B} mat", + "Using {a}, set {A} on the {B} colored mat", + "Put {A} on the {B} mat", + "Lift {A} to the {B} mat using {a}", + "Place {A} onto the {B} colored mat", + "Set {A} down on the {B} mat", + "With {a}, position {A} on the {B} mat", + "Transfer {A} to the {B} mat", + "Move {A} with {a} to the {B} mat", + "Drop {A} carefully on the {B} mat", + "Place {A} on the {B} mat using {a}", + "Lift {A} and drop it onto {B} mat", + "Shift {A} manually to the {B} mat", + "Move {A} to the {B} mat with {a}", + "Grab {A} and stick it onto {B} mat", + "Use {a} to grab {A} and place it on {B} mat", + "Pick {A} up and position it on {B} mat", + "Carry {A} and drop it onto the {B} mat", + "Use {a} to shift {A} onto the {B} mat", + "Pick {A} with {a} and place it on {B} mat", + "Use {a} to grab {A} and move it", + "Set {A} down on the {B} mat", + "Pick up {A} and place it on {B} mat", + "Grab {A} using {a} and shift it to {B}", + "Relocate {A} to the {B} colored mat", + "Use {a} to place {A} onto the {B} mat", + "Shift {A} to the {B} mat", + "Pick up {A} with {a} and set it on {B}", + "Carry {A} to the {B} colored mat", + "With {a}, move {A} to the {B} mat" + ], + "unseen": [ + "Move {A} to the {B} mat.", + "Place {A} on the {B} colored mat.", + "Use {a} to move {A} to {B} mat.", + "Place {A} on the {B} colored mat.", + "Use {a} to move {A} to the {B} mat", + "Move {A} to the {B} colored mat", + "Grab {A} and set it on {B} mat", + "Use {a} to move {A} to {B} mat", + "Move {A} to the {B} mat", + "Place {A} on the {B} mat" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/open_laptop.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/open_laptop.json new file mode 100644 index 00000000..cf020311 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/open_laptop.json @@ -0,0 +1,69 @@ +{ + "full_description": "use one arm to open the laptop", + "schema": "{A} notifies the laptop. {a} notifies the arm used to open the laptop.", + "preference": "num of words should not exceed 6!!!!!. Degree of detail avg is 2.", + "seen": [ + "Raise {A} lid with {a}.", + "Fully open the {A}.", + "Use {a} to open {A}.", + "Lift {A} lid completely.", + "Raise {A} lid using {a}.", + "Open {A} entirely.", + "Fully lift {A} using {a}.", + "Open {A} completely.", + "Use {a} to raise {A}.", + "Lift {A} to open.", + "Open {A} carefully with {a}.", + "Lift and open {A}.", + "Pull {A} upward lightly.", + "Raise {A} using {a}.", + "Pull {A} open with {a}.", + "Gently open {A}.", + "Use {a} to raise {A}.", + "Lift {A} slightly.", + "Raise and open {A}.", + "Use {a} to open {A}.", + "Use {a} to open {A}", + "Raise the top of {A}", + "Gently lift {A} using {a}", + "Pull open {A}", + "Carefully open {A} using {a}", + "Lift {A} to open it up", + "Use {a} to lift {A}", + "Pull up the lid of {A}", + "Open {A} lifting with {a}", + "Raise {A} to expose the screen", + "Lift {A}'s lid upwards.", + "Use {a} to raise {A}.", + "Raise {A} using {a}.", + "Grip and lift {A}'s lid.", + "Hold {A} steady, lift lid.", + "Gently raise {A} using {a}.", + "Open {A} by lifting its lid.", + "Lift {A}'s lid using {a}.", + "With {a}, open {A}'s lid.", + "Pull the lid of {A} upward.", + "Raise the lid of {A}.", + "Use {a} to open {A}.", + "Lift {A} open with {a}.", + "Raise {A}'s top section.", + "Pull {A}'s lid upward.", + "Gently open {A} upward.", + "Push {A}'s lid upward using {a}.", + "Raise {A} using {a} firmly.", + "Slide {A} open with {a}.", + "Pull the screen of {A}." + ], + "unseen": [ + "Open {A} using {a}.", + "Lift {A}'s lid fully.", + "Grab {A} and open it.", + "Lift {A} using {a}.", + "Lift {A} using {a}", + "Open {A} with care", + "Pull up {A}'s lid.", + "Open {A} with one motion.", + "Lift {A}'s lid gently.", + "Open {A} carefully using {a}." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/open_microwave.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/open_microwave.json new file mode 100644 index 00000000..90a02418 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/open_microwave.json @@ -0,0 +1,69 @@ +{ + "full_description": "Use one arm to open the microwave.", + "schema": "{A} notifies the microwave, {a} notifies the arm to open the microwave", + "preference": "num of words should not exceed 15", + "seen": [ + "Use {a} to pull open the door of {A}.", + "Pull the handle to open {A}.", + "Grab the handle and open {A} with {a}.", + "Open {A} by grasping its handle.", + "Pull open the door of {A} using {a}.", + "Pull the door of {A} to open it.", + "Use {a} to grab and open {A}.", + "Open {A} by pulling its door with {a}.", + "Open {A} by pulling its handle.", + "Pull the handle of {A} to open it.", + "Pull the door to open {A}.", + "Grab the handle and open {A}.", + "Use {a} to pull {A} open.", + "Engage {a} to open {A}.", + "Pull and open {A}.", + "Pull the microwave door to open.", + "Open {A} by pulling with {a}.", + "Use the handle to open {A}.", + "Engage {A} using {a} to open.", + "Pull the handle to open {A}.", + "Open {A} using {a}.", + "Get {A} open.", + "Access the inside by opening {A}.", + "Activate the door to open {A}.", + "Pull to open {A}.", + "Grip {A}'s handle and pull it open.", + "Handle {A} and open it with {a}.", + "Use {a} to engage {A}'s opening mechanism.", + "Engage {A}'s handle and open it.", + "Utilize {a} to pull {A} open.", + "Open the microwave door by using {a}.", + "Grip and pull the microwave door open.", + "Pull the handle to open the microwave.", + "Use {a} to open the microwave door.", + "Open the microwave door by pulling it.", + "Lift the microwave door with {a} to open it.", + "Use {a} to pull the microwave door open.", + "Pull open the microwave using its handle.", + "Grab the handle and open the microwave door.", + "Use {a} to grip and open the microwave.", + "Open {A} by pulling its handle.", + "Pull the {A}'s handle using {a}.", + "Grasp {A}'s handle and pull to open.", + "Pull the handle of {A} with {a}.", + "Open {A} by pulling on its handle.", + "Use {a} to grab and pull {A}.", + "Pull the handle to open {A}.", + "Grab and pull {A} using {a}.", + "Pull {A}'s handle to open it.", + "Use {a} to open {A} by pulling." + ], + "unseen": [ + "Locate {A} and open it using {a}.", + "Open {A} by pulling its door handle.", + "Move {a} and open {A}.", + "Open {A} using {a}.", + "Use {a} to open {A}.", + "Pull {A} open with {a}.", + "Use {a} to pull open the microwave.", + "Pull the microwave door open with {a}.", + "Grab {A}'s handle and pull to open it.", + "Use {a} to pull the handle of {A}." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/pick_diverse_bottles.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/pick_diverse_bottles.json new file mode 100644 index 00000000..467fabcb --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/pick_diverse_bottles.json @@ -0,0 +1,69 @@ +{ + "full_description": "pick up one bottle with one arm, and pick up another bottle with the other arm", + "schema": "{A} notifies the left bottle to be catched,{B} notifies the right bottle to be catched. arm comes as a literal here", + "preference": "num of words should not exceed 10. Degree of detail avg 5", + "seen": [ + "Lift {A} and also take hold of {B}.", + "Catch {A}, then grab {B} too.", + "Secure both {A} and {B} quickly.", + "Hold onto {A}, then pick up {B}.", + "Use one arm for {A}, the other for {B}.", + "Let each arm grab either {A} or {B}.", + "Assign one arm to catch {A}, one for {B}.", + "Use both arms to lift {A} and {B}.", + "Grab {A} with one arm, {B} with the other.", + "Each arm should pick up one: {A}, then {B}.", + "Pick up {A} and then pick up {B}.", + "Hold {A} with one arm, hold {B} with the other.", + "Catch {A}, then catch {B}.", + "Grab {A}, pick up {B}.", + "Use each arm to lift {A} and {B}.", + "Hold {A} in one arm, {B} in the other.", + "Secure {A}, then secure {B}.", + "Use both arms to pick up {A} and {B}.", + "Pick up both {A} and {B}.", + "Catch {A}, grab {B} next.", + "Grab {A} with one arm, {B} with the other.", + "Collect {A}, then pick up {B}.", + "Lift {A} and {B} with separate arms.", + "Catch both {A} and {B} quickly.", + "Use one arm for {A}, another for {B}.", + "Secure {A} and {B} in succession.", + "With both arms, pick up {A}, then {B}.", + "Grasp {A} first, then take {B}.", + "Assign one arm to each of {A} and {B}.", + "Pick both {A} and {B} one after another.", + "Grab {A} using one arm and {B} using the other.", + "Lift both bottles, {A} with one arm, {B} with the other.", + "One arm grabs {A}; the other arm grabs {B}.", + "With two arms, catch both {A} and {B}.", + "Pick up both {A} and {B} together.", + "Grab and hold {A}, then grab {B}.", + "Catch {A}, then take hold of {B}.", + "Pick up {A}, then pick up {B}.", + "Lift {A}, continue and pick up {B}.", + "Hold onto {A} and then get {B}.", + "Lift both {A} and {B} simultaneously.", + "Hold both {A} and {B}.", + "Grab {A} and pick up {B}.", + "Take hold of {A} and {B}.", + "Use one arm for {A} and the other for {B}.", + "Catch {A} with one arm, {B} with another arm.", + "Use both arms to pick up {A} and {B}.", + "Catch {A} in one arm and {B} in the other arm.", + "Hold {A} in one arm, catch {B} with the other arm.", + "Grab {A} using an arm and {B} with the other arm." + ], + "unseen": [ + "Grab {A} and {B} at the same time.", + "Pick up both {A} and {B} together.", + "Grab {A} with one arm, grab {B} with the other.", + "Lift {A} using one arm and lift {B} using the other.", + "Use each arm to grab {A} and {B}.", + "Pick up {A} and {B} together.", + "Pick up {A} with one arm, grab {B} with the other.", + "Use each arm to lift a bottle: {A}, then {B}.", + "Grab {A} and grab {B} at once.", + "Pick up {A} and take {B} together." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/pick_dual_bottles.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/pick_dual_bottles.json new file mode 100644 index 00000000..f32cf779 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/pick_dual_bottles.json @@ -0,0 +1,69 @@ +{ + "full_description": "pick up one bottle with one arm, and pick up another bottle with the other arm", + "schema": "{A} notifies one bottle to be catched,{B} notifies the other bottle to be catched. arm comes as a literal here", + "preference": "num of words should not exceed 10.Degree of detail avg 5", + "seen": [ + "Take {A} with one arm, hold {B} too.", + "Use each arm to grab {A} and {B}.", + "Lift {A} in one hand and {B} in the other.", + "Pick up {A} and {B} using separate hands.", + "Catch {A} with one arm, then grab {B}.", + "Take {A}, then use the other arm for {B}.", + "Hold {A} in one hand and {B} in another.", + "Grab {A} with one hand, then reach for {B}.", + "Lift {A} with one arm and {B} with the other.", + "Hold {A} and {B} using both arms separately.", + "Hold {A} and {B} using both arms.", + "Grab {A} first, then grab {B} second.", + "Catch {A}, then catch {B} after.", + "Pick {A} with one hand, pick {B} next.", + "Reach for {A}, then grab {B}.", + "Grasp {A} and {B} with both arms.", + "Use each arm to hold {A} and {B}.", + "Catch {A} first, then catch {B}.", + "Pick {A} with one hand, {B} with another.", + "Grab {A} with an arm, grab {B} next.", + "Lift {A} and {B} simultaneously with both arms.", + "Catch {A}, then catch {B} without mentioning arms.", + "Raise {A} first, then grab {B} next.", + "Hold {A} in one hand, then hold {B} in the other.", + "Handle {A} and {B} together without arm details.", + "Pick {A} first, then pick {B} without mentioning arms.", + "Catch {A} with one hand, catch {B} with the other.", + "Grab and lift {A}, then lift {B} next.", + "Lift {A} in one arm and {B} in the other.", + "Handle {A}, then handle {B} without arm specifics.", + "Secure {A} in one hand, {B} in other.", + "Hold {A} and {B}, one in each hand.", + "Pick up {A} and {B} together.", + "Grab {A} and {B} one by one.", + "Lift both {A} and {B} bottles.", + "Grab the bottles {A} and {B}.", + "Pick {A} first and then {B}.", + "Hold {A} and {B} separately.", + "Lift bottle {A}, then bottle {B}.", + "Catch {A} in one hand and {B} too.", + "Hold {A} in one hand, {B} in another.", + "Grab {A} with one arm, {B} with the other.", + "Secure {A} and {B} using separate hands.", + "Pick {A} in one arm and {B} in the other.", + "Lift {A} and {B} together with both hands.", + "Use one arm for {A} and the other for {B}.", + "Grasp {A} and {B} at the same time.", + "Catch {A} in one arm, {B} in the other.", + "Hold onto {A} and {B} using both hands.", + "Use separate arms to pick {A} and {B}." + ], + "unseen": [ + "Pick {A}, then pick {B}.", + "Hold {A} in one hand, {B} in the other.", + "Grab {A} and {B} with arms.", + "Use one arm to grab {A}, the other for {B}.", + "Pick up {A} and {B} using both arms.", + "Grab {A} with one arm, grab {B} with the other.", + "Pick {A} with one arm, {B} with the other.", + "Lift {A} and {B} using both arms.", + "Pick up {A} and {B} simultaneously.", + "Take hold of {A} and {B} at once." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_a2b_left.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_a2b_left.json new file mode 100644 index 00000000..4d953d59 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_a2b_left.json @@ -0,0 +1,69 @@ +{ + "full_description": "use appropriate arm to place object A on the left of object B", + "schema": "{A} notifies the object A, {B} notifies the object B, {a} notifies the arm to grab the object A", + "preference": "num of words should not exceed 10.STRESS THE 'left'.", + "seen": [ + "Grab {A} using {a} and place left of {B}.", + "Place {A} on the left of {B}.", + "With {a}, position {A} left of {B}.", + "Position {A} to the left of {B}.", + "Using {a}, move {A} left of {B}.", + "Move {A} to the left of {B}.", + "Use {a} and place {A} left of {B}.", + "Set {A} left of {B}.", + "With {a}, set {A} on the left of {B}.", + "Carefully position {A} to the left of {B}.", + "Place {A} left of {B} using {a}", + "Move {A} to the left of {B}", + "Put {A} on the left side of {B}", + "Grab {A} with {a} and set left of {B}", + "Shift {A} leftward to the side of {B}", + "Use {a} to move {A} to {B}'s left", + "Adjust {A} to rest on {B}'s left", + "Use {a} to position {A} left of {B}", + "Place {A} carefully to {B}'s left", + "Grab {A} using {a} and move left of {B}", + "Use {a}, move {A} left of {B}", + "Position {A} to the left of {B}", + "Place {A} exactly to the left of {B}", + "Using {a}, put {A} on left of {B}", + "Set {A} carefully on the left of {B}", + "Move {A} to the left side of {B}", + "Place {A} to rest left of {B} using {a}", + "With {a}, position {A} left of {B}", + "Set {A} directly to the left of {B}", + "Using {a}, place {A} on the left of {B}", + "Move {A} to the left of {B}.", + "Grab {A} with {a} and shift it left of {B}.", + "Place {A} directly to the left of {B}.", + "With {a}, move {A} to the left side of {B}.", + "Shift {A} and position it left of {B}.", + "Use {a} to move {A} to the left of {B}.", + "Set {A} on the left side of {B}.", + "Grab {A} using {a} and place it left of {B}.", + "Position {A} carefully to the left of {B}.", + "Using {a}, set {A} to the left of {B}.", + "Put {A} to the left of {B}", + "Use {a} to set {A} left of {B}", + "Drop {A} on the left side of {B}", + "With {a}, stick {A} left of {B}", + "Set {A} to the left of {B}", + "Place {A} left of {B} using {a}", + "Stick {A} on the left of {B}", + "Use {a} to put {A} left of {B}", + "Drop {A} to the left of {B}", + "With {a}, place {A} left of {B}" + ], + "unseen": [ + "Use {a} to put {A} left of {B}.", + "Set {A} on the left side of {B}.", + "Set {A} to the left of {B}", + "Position {A} on the left of {B}", + "Grab {A} and place left of {B}", + "Set {A} down to the left of {B}", + "Pick {A} and set it left of {B}.", + "Use {a} to place {A} left of {B}.", + "Set {A} on the left side of {B}", + "Place {A} to the left of {B} using {a}" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_a2b_right.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_a2b_right.json new file mode 100644 index 00000000..bbdb6f38 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_a2b_right.json @@ -0,0 +1,69 @@ +{ + "full_description": "use appropriate arm to place object A on the right of object B", + "schema": "{A} notifies the object A, {B} notifies the object B, {a} notifies the arm to grab the object A", + "preference": "num of words should not exceed 10.STRESS THE 'right'.", + "seen": [ + "Use {a} to set {A} right of {B}.", + "Position {A} to the right of {B}.", + "Move {A} to {B}'s right position.", + "Grab {A} with {a} and place it right of {B}.", + "Ensure {A} is placed on the right of {B}.", + "Grab {A} using {a} and set it to {B}'s right.", + "Move {A} and place it carefully to {B}'s right.", + "Take {A} with {a} and position it right of {B}.", + "Set {A} neatly on the right side of {B}.", + "Grab {A} using {a}, then move it to {B}'s right.", + "Set {A} on the right side of {B}.", + "Use {a} to position {A} to {B}'s right.", + "Move {A} and place it to {B}'s right.", + "Put {A} using {a} on the right of {B}.", + "Set {A} right next to {B}'s right side.", + "Grasp {A} with {a} and move it to {B}'s right.", + "Place {A} directly on {B}'s right.", + "Using {a}, position {A} to the right of {B}.", + "Set {A} carefully on {B}'s right side.", + "Grab {A} using {a} and place it right of {B}.", + "Use {a} to place {A} right of {B}.", + "Position {A} directly to the right of {B}.", + "Move {A} and set it to {B}'s right.", + "With {a}, move {A} to {B}'s right.", + "Set {A} down on {B}'s right side.", + "Bring {A} to the right position of {B}.", + "Using {a}, position {A} to the right of {B}.", + "Place {A} precisely to the right of {B}.", + "Move {A} using {a} to {B}'s right position.", + "Set {A} properly on the right of {B}.", + "Set {A} on {B}'s right side", + "Position {A} to {B}'s right using {a}", + "Shift {A} to the right of {B}", + "Move {A} right of {B} with {a}", + "Stick {A} on {B}'s right side", + "Place {A} using {a} at {B}'s right", + "Position {A} at the right side of {B}", + "Move {A} to {B}'s right using {a}", + "Put {A} at the right of {B}", + "Set {A} using {a} at {B}'s right", + "Use {a} to place {A} right of {B}.", + "Position {A} to the right of {B}.", + "Move {A} to the right beside {B}.", + "Ensure {A} is on the right of {B}.", + "Direct {a} to place {A} right of {B}.", + "Put {A} exactly to the right of {B}.", + "Use {a} to set {A} right of {B}.", + "Place {A} on the right of {B}.", + "Grab {A} with {a} and set right of {B}.", + "Make sure {A} is placed right of {B}." + ], + "unseen": [ + "Put {A} to the right of {B}.", + "Place {A} on {B}'s right side.", + "Put {A} to the right of {B}.", + "Grab {A} and place it right of {B}.", + "Place {A} on the right of {B}.", + "Set {A} to the right of {B}.", + "Put {A} to the right of {B}", + "Place {A} rightward of {B} using {a}", + "Put {A} to the right of {B}.", + "Set {A} on the right of {B}." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_bread_basket.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_bread_basket.json new file mode 100644 index 00000000..ec4a91a6 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_bread_basket.json @@ -0,0 +1,78 @@ +{ + "full_description": "if there is one bread on the table, use one arm to grab the bread and put it in the basket, if there are two breads on the table, use two arms to simultaneously!!! grab up two breads and put them in the basket", + "schema": "{A} notifies the basket, {B} notifies the first bread(or the only bread if there is only one bread), {C} notifies the second bread(if there are two breads), {a} notifies the arm to grab the bread(may be left, right, or dual)", + "preference": "num of words should not exceed 10. Degree of detail avg is six. NOTE!! 50% of the instructions are about one bread scenario, 50% of the instructions are about two breads scenario", + "seen": [ + "Pick up {B} and put it in {A}.", + "Use {a} to grab {B} and drop it inside {A}.", + "Grab {B} with one hand and set it in {A}.", + "Pick up both {B} and {C}, then place them in {A}.", + "Simultaneously grab {B} and {C} using {a}, then drop them in {A}.", + "Take {B} and {C} together and place them into {A}.", + "Lift {B} and {C} at once with {a}, then set them in {A}.", + "Pick both breads and place them into {A}.", + "Use {a} to grab both breads, then put them in {A}.", + "Grab {B} and {C} quickly and drop them into {A}.", + "Pick up {B} and drop it in {A}.", + "Use both {a} to grab {B} and {C}.", + "Pick {B} and {C} and set them in {A}.", + "Use {a} to place {B} and {C} into {A}.", + "Pick {B} and put it into {A}.", + "Grab {B} with {a} and drop it in {A}.", + "Grab two breads {B} and {C} and place in {A}.", + "Simultaneously use {a} to drop {B} and {C} in {A}.", + "Pick {B} and move it to {A}.", + "Grab both {B} and {C} with {a} and place in {A}.", + "Lift {B} and transfer to {A}.", + "Move {B} to {A} using one arm.", + "Grab {B}, drop it into {A}.", + "Use two arms to grab {B} and {C}.", + "Pick {B} and {C}, place them in {A}.", + "Simultaneously grab {B} and {C}, drop in {A}.", + "Move {B} and {C} at once into {A}.", + "With both arms, grab {B} and {C}.", + "Shift {B} and {C} together to {A}.", + "Put {B} and {C} into {A} using two arms.", + "Lift {B} and set it in {A}.", + "Put {B} into {A} using an arm.", + "Take {B} and {C} then place in {A}.", + "Use two arms and set {B}, {C} in {A}.", + "Grab both {B} and {C}, drop into {A}.", + "Lift {B} and {C} with two arms, put in {A}.", + "Put {B} into {A} after grabbing it.", + "Grab {B} with an arm and set in {A}.", + "Take {B} and {C}, place them inside {A}.", + "Use both arms to move {B}, {C} to {A}.", + "Use {a} to grab {B} for {A}", + "Drop {B} into {A}", + "Simultaneously grab {B} and {C}", + "Move {B} and {C} to {A}", + "Use {a} to pick and place {B} {C}", + "Shift {B} and {C} into {A}", + "Pick {B} and {C} for the {A}", + "Grab {B} for {A} with {a}", + "Take {B} and {C} to {A}", + "Place {B} and {C} in {A} using {a}" + ], + "unseen": [ + "Grab {B} and drop it into {A}.", + "Use {a} to pick up {B}, then place it in {A}.", + "Grab {B} and put it in {A}.", + "Use {a} to pick {B} and place in {A}.", + "Pick {B} and place it in {A}.", + "Use one arm to grab {B}, drop in {A}.", + "Grab {B} and drop it into {A}.", + "Grab {B} with one arm, place in {A}.", + "Pick {B} and drop it in {A}", + "Place {B} into {A} using {a}", + "Grab {B} and {C} and drop them into {A}.", + "Use {a} to pick up {B} and {C}, then place them in {A}.", + "Grab {B} and {C} and put them in {A}.", + "Use {a} to pick {B} and {C} and place in {A}.", + "Pick {B} and {C} and place them in {A}.", + "Use one arm to grab {B} and {C}, drop in {A}.", + "Grab {B} and {C} and drop it into {A}.", + "Grab {B} and {C} with two arms, place in {A}.", + "Place {B} and {C} into {A} using {a}" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_bread_skillet.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_bread_skillet.json new file mode 100644 index 00000000..492a047d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_bread_skillet.json @@ -0,0 +1,69 @@ +{ + "full_description": "if there is one bread on the table, use one arm to grab the bread and put it into the skillet", + "schema": "{A} notifies the skillet, {B} notifies the bread, {a} notifies the arm to grab the bread", + "preference": "num of words should not exceed 10. Degree of detail avg is 6", + "seen": [ + "Grab {B} and place it inside {A}", + "Pick up {B} with {a} and drop it in {A}", + "Place {B} from the table into {A}", + "Lift {B} using {a} and move it to {A}", + "Move {B} into {A} from the table", + "Pick {B} with {a} and set it in {A}", + "Transfer {B} into {A} from the table", + "Grab {B} using {a} and place it in {A}", + "Take {B} and place it inside {A}", + "Use {a} to pick {B} and put it in {A}", + "Use {a} to grab {B} for {A}", + "Place {B} into {A} using {a}", + "Take {B} and set it into {A}", + "Move {B} to {A} using {a}", + "Pick up {B} and put it in {A}", + "Grab {B} with {a} and place in {A}", + "Use {a} to pick {B} up for {A}", + "Set {B} into {A} after grabbing", + "Put {B} into {A} after grabbing it", + "Take {B} with {a} and drop into {A}", + "Move {B} from table into {A}", + "Use {a} to place {B} into {A}", + "Lift {B} and drop it into {A}", + "Pick {B} and transfer it to {A}", + "Take {B} off the table into {A}", + "Use {a} to move {B} into {A}", + "Grab {B} and drop it into {A}", + "Use {a} to grab {B} for {A}", + "Take {B} and place it inside {A}", + "Use {a} to pick {B} for {A}", + "Pick up {B} and place it in {A}.", + "Use {a} to set {B} into {A}.", + "Take {B} and put it in {A}.", + "Place {B} into {A} with {a}.", + "Grab {B} from the table and set it in {A}.", + "Pick {B} up using {a} and drop it in {A}.", + "Put {B} into {A}.", + "Grab {B} and place it inside {A}.", + "Use {a} to move {B} and set it into {A}.", + "Pick {B} off the table and place it in {A}.", + "Take {B} and drop it into {A}.", + "Use {a} to grab {B} and set it in {A}.", + "Grab {B} and stick it in {A}.", + "Use {a} to pick {B} and place it into {A}.", + "Lift {B} and drop it into {A}.", + "Use {a} to take {B} and move it into {A}.", + "Pick {B} and set it into {A}.", + "Use {a} to lift {B} and place it into {A}.", + "Take {B} and put it in {A}.", + "Use {a} to stick {B} into {A}." + ], + "unseen": [ + "Take {B} and put it in {A}", + "Use {a} to grab {B} and set it in {A}", + "Grab {B} and place it in {A}", + "Pick up {B} and drop it into {A}", + "Pick {B} up and place it in {A}", + "Grab {B} and set it in {A}", + "Grab {B} and drop it into {A}.", + "Move {B} to {A} using {a}.", + "Grab {B} and place it into {A}.", + "Use {a} to move {B} to {A}." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_burger_fries.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_burger_fries.json new file mode 100644 index 00000000..dde91159 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_burger_fries.json @@ -0,0 +1,69 @@ +{ + "full_description": "Use dual arm to pick the hamburg and frenchfries and put them onto the tray.", + "schema": "{A} notifies the hamburg, {B} notifies the tray, {C} notifies the frenchfries", + "preference": "num of words should not exceed 15", + "seen": [ + "Use both arms to move {A} and {C} to {B}.", + "Lift {A} and {C}, placing them neatly on {B}.", + "Place {A} and {C} on {B} after picking them up.", + "Take {A} and {C} with both arms and set on {B}.", + "Move {A} and {C} to {B} using both arms.", + "Pick up {A} and {C}, then put both on {B}.", + "Using both arms, grab {A} and {C} for {B}.", + "Set {A} and {C} down on {B} after picking them.", + "Grab {A} with one arm and {C} with the other for {B}.", + "Take {A} and {C} to {B} and set them there.", + "Grab {A}, grab {C}, and set both onto {B}.", + "With dual arms, move {A} and {C} onto {B}.", + "Place {A} and {C} together onto {B}.", + "Lift {A}, lift {C}, then set both on {B}.", + "Move {A} and {C} with dual arms to {B}.", + "Take {A} and {C}, place them onto {B}.", + "Use both arms to transfer {A} and {C} to {B}.", + "Pick up {A}, then {C}, and place them onto {B}.", + "Using dual arms, move {A} and {C} onto {B}.", + "Grab {A} and {C}, use both arms to set on {B}.", + "Use dual arms to place {A} and {C} onto {B}", + "Put {A} and {C} on {B} using both arms", + "Lift {A} and {C}, position them on {B}", + "Pick up {A}, grab {C}, place them on {B}", + "Move {A} and {C} to {B} with both arms", + "Place {A} and {C} onto {B} using dual arms", + "Grab {A}, pick {C}, then set them on {B}", + "Transfer {A} and {C} to {B} with both arms", + "Pick up {A} and {C}, then put them on {B}", + "Use dual arms to move {A} and {C} onto {B}", + "Use both arms to move {A} and {C} to {B}.", + "Set {A} and {C} on {B} using two arms.", + "Grab {A} and {C}, then put them on {B}.", + "Move {A} and {C} onto {B} with both arms.", + "Take {A}, take {C}, and place them onto {B}.", + "With two arms, pick {A} and {C} for {B}.", + "Lift {A} and {C} and position them on {B}.", + "Using both arms, set {A} and {C} onto {B}.", + "Pick {A} and {C}, and set them on {B}.", + "Position {A} and {C} on {B} with both arms.", + "Place {A} and {C} onto {B} after picking them up.", + "Pick {A}, {C}, and place them on {B} using both arms.", + "Grab {A}, then {C}, and set them on {B}.", + "Using both arms, pick {A} and {C} and drop them on {B}.", + "Pick up {A} and {C} one by one and place on {B}.", + "Place both {A} and {C} on {B} after grabbing them with both arms.", + "Grab {A} and {C} and position them on {B}.", + "Using dual arms, lift {A} and {C} and drop them on {B}.", + "Pick {A}, then {C}, and place them on {B}.", + "Use dual arms to grab {A} and {C} and set them on {B}." + ], + "unseen": [ + "Pick {A} and {C}, then place them on {B}.", + "Grab {A} and {C} together, setting them on {B}.", + "Pick up {A} and {C}, then place on {B}.", + "Use both arms to place {A} and {C} on {B}.", + "Grab {A} and {C}, then set them on {B}", + "Pick up {A} and {C} together, then drop on {B}", + "Pick {A} and {C}, place them on {B}.", + "Place {A} and {C} onto {B}.", + "Pick up {A} and {C} and place them on {B}.", + "Use both arms to grab {A} and {C} and set them on {B}." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_can_basket.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_can_basket.json new file mode 100644 index 00000000..262de474 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_can_basket.json @@ -0,0 +1,69 @@ +{ + "full_description": "use one arm to pick up the can, put it into the basket, and use another arm to lift the basket", + "schema": "{A} notifies the can, {B} notifies the basket, {a} notifies the arm to pick up the can, the arm to pick up the basket use literal 'the other arm' or 'another arm'", + "preference": "num of words should not exceed 15. Degree of detail avg 8.If {a} is mentioned in one description, 'the other arm' or 'another arm' must be mentioned in another description.Or otherwise both arm should not be mentioned. BUT {A} and {B} should always be mentioned in each description", + "seen": [ + "Place {A} in {B} and then lift {B}.", + "Pick {A} with {a}, drop it in {B}, lift {B} with another arm.", + "Pick {A}, set it in {B}, then lift {B}.", + "Use the same arm to grab {A}, place in {B}, and then lift {B}.", + "Take {A} in {a}, place it into {B}, another arm lifts {B}.", + "Grab and drop {A} in {B}, lift {B}.", + "Pick up {A} with {a}, place into {B}, the other arm raises {B}.", + "Take {A}, set it into {B}, and raise {B}.", + "Use {a} to grab {A} and place in {B}, lift {B} with another arm.", + "Pick up {A}, place it in {B}, and then lift {B}.", + "Place {A} in {B} and then use another arm to lift {B}.", + "Pick up {A}, set it inside {B}, then lift {B}.", + "Grab {A} using {a}, drop {A} into {B} and lift {B}.", + "Use {a} for {A}, place {A} in {B}, and lift {B}.", + "Take {A} and put it in {B}, then lift {B}.", + "Drop {A} into {B} using {a}, then lift {B} with another arm.", + "Pick up {A}, place it into {B}, then lift {B}.", + "Use {a} to grab {A}, place it in {B}, and lift {B}.", + "Grab {A}, drop it into {B}, then use another arm for {B}.", + "Place {A} inside {B}, then lift {B}.", + "Hold {A} using {a}, place it in {B}, and lift {B}.", + "Lift {A}, drop it into {B}, then carry {B}.", + "Grab {A} with {a}, put it into {B}, and lift {B} with the other arm.", + "Lift {A}, place it in {B}, pick up {B} afterward.", + "Pick {A} with one arm, place it in {B}, and lift {B} using another arm.", + "Grab {A}, set it into {B}, then raise {B}.", + "Pick up {A}, put it into {B}, and lift {B}.", + "Use {a} to hold {A}, drop {A} into {B}, and lift {B}.", + "Grab {A}, place it in {B}, and raise {B} using the other arm.", + "Lift {A}, put it in {B}, and carry {B} afterward.", + "Pick up {A}, put it in {B}, and lift {B}.", + "Lift {A} with {a}, set it in {B}, and lift with another arm.", + "Take {A}, place it into {B}, then lift {B}.", + "Grab {A}, drop it into {B}, and raise {B} with the other arm.", + "Pick {A}, set it in {B}, and lift {B} upward.", + "Use {a} to grab {A}, put it into {B}, and lift {B} with another arm.", + "Lift {A}, place it in {B}, and raise {B}.", + "Take {A} with {a}, drop it into {B}, and lift {B} using another arm.", + "Grab {A}, place it in {B}, and lift {B} upwards.", + "Pick {A} using {a}, drop it in {B}, and lift {B} with the other arm.", + "Grab {A}, place it in {B}, and then lift {B}.", + "Pick up {A} using {a}, drop it into {B}, and lift {B} with the other arm.", + "Lift {A}, set it in {B}, and elevate {B}.", + "Use {a} to pick {A}, place it inside {B}, then lift {B}.", + "Grab {A}, drop it into {B}, and raise {B}.", + "Pick {A} with {a}, set it in {B}, and lift {B} using another arm.", + "Lift {A}, place it inside {B}, and elevate {B}.", + "Use {a} to take {A}, drop it in {B}, and lift {B}.", + "Grab {A}, put it in {B}, then raise {B}.", + "Use {a} to grab {A}, place it into {B}, and lift {B} using the other arm." + ], + "unseen": [ + "Grab {A}, put it in {B}, lift {B}.", + "Use {a} to pick up {A}, put it into {B}, the other arm lifts {B}.", + "Grab {A} and put it in {B}, then lift {B}.", + "Use {a} to pick {A}, drop it into {B}, then lift {B}.", + "Grab {A}, drop it into {B}, lift {B} with another arm.", + "Pick {A}, put it in {B}, then lift {B}.", + "Grab {A}, drop it in {B}, and lift {B}.", + "Use {a} to pick {A}, place it into {B}, and lift the basket.", + "Pick up {A}, place it in {B}, then lift {B}.", + "Use {a} to grab {A}, put it in {B}, and lift {B}." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_cans_plasticbox.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_cans_plasticbox.json new file mode 100644 index 00000000..92f2df0f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_cans_plasticbox.json @@ -0,0 +1,69 @@ +{ + "full_description": "Use dual arm to pick and place cans into plasticbox", + "schema": "{A} notifies the left can, {B} notifies the plasticbox, {C} notifies right can", + "preference": "num of words should not exceed 15", + "seen": [ + "Use both arms to move {A} and {C} into {B}.", + "Lift {A}, put it in {B}, then handle {C} similarly.", + "With both arms, transfer {A} and {C} into {B}.", + "Pick {A}, place it inside {B}, follow the same for {C}.", + "Move {A} and {C} one at a time into {B}.", + "Use dual arms to pick {A} and {C}, placing both in {B}.", + "Place {A} in {B}, follow with {C} using each arm.", + "Transfer {A} to {B}, then transfer {C} to {B}.", + "First move {A} to {B}, then move {C} into {B}.", + "Use your arms to set {A} and {C} gently into {B}.", + "Move {A} to {B} and repeat with {C}.", + "Use both arms to place {A} and {C} inside {B}.", + "Pick {A}, place it in {B}, then pick {C} and place it in {B}.", + "Grip {A} and insert it into {B}, then repeat for {C}.", + "Move {A} and {C} into {B} using separate arms.", + "Transfer {A} to {B}, then transfer {C} to {B}.", + "Place {A} and {C} into {B} using both arms.", + "Use arms to set {A} and {C} into {B}.", + "Pick and drop {A} and {C} into {B}.", + "Lift {A}, place it in {B}, then repeat for {C}.", + "Use both arms to move {A} and {C} into {B}", + "Lift {A} and {C}, then set them inside {B}", + "Pick {A} and {C} using both arms and put them in {B}", + "Place {A} and {C} into {B} with dual arms", + "Move {A} and {C} together into {B}", + "Transfer {A} and {C} into {B} using dual arms", + "Pick and drop {A} and {C} into {B} together", + "Using dual arms, place {A} and {C} inside {B}", + "Lift {A} and {C}, and stick them into {B}", + "Drop {A} and {C} into {B} with both arms", + "Use both arms to grab {A} and {C}, place them in {B}.", + "Grab {A}, insert it into {B}, then grab {C} and repeat.", + "Place {A} and {C} in {B} using both arms.", + "Lift {A} into {B}, then {C} to {B} without delay.", + "Pick up {A} using one arm, set it in {B}, repeat for {C}.", + "Use arms to pick {A}, drop it in {B}, repeat for {C}.", + "Identify {A}, place it in {B}, do the same for {C}.", + "Grab {A}, transfer it to {B}, then repeat with {C}.", + "Both arms lift {A}, drop into {B}, repeat for {C}.", + "Pick {A} and {C}, put them together inside {B}.", + "Use both arms to place {A} and {C} into {B}.", + "First pick {A}, place it in {B}, then repeat for {C}.", + "Place {A} into {B}, then lift {C} and set it into {B}.", + "Use the arms to transfer {A} and {C} into {B}.", + "Move {A} into {B}, then pick and drop {C} into the same box.", + "Use both arms, position {A} and {C} within {B}.", + "Begin with {A}, place it in {B}, finish with {C} into {B}.", + "Utilize the arms to deposit both {A} and {C} into {B}.", + "Transfer {A} into {B}, then position {C} within {B}.", + "Employ both arms to pick {A} and {C}, and place them inside {B}." + ], + "unseen": [ + "Pick {A}, place it into {B}, then repeat with {C}.", + "Grab {A}, drop it in {B}, and do the same for {C}.", + "Pick {A} and place it into {B}. Then do the same for {C}.", + "Grab {A}, drop it into {B}. Repeat for {C}.", + "Grab {A} and {C} and place them in {B}", + "Pick up {A} and {C}, drop them together into {B}", + "Pick {A}, set it in {B}, then repeat with {C}.", + "Lift {A} and {C}, drop both into {B}.", + "Move {A} into {B}, then move {C} into {B}.", + "Grab {A} and drop it into {B}, then do the same for {C}." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_container_plate.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_container_plate.json new file mode 100644 index 00000000..724de06d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_container_plate.json @@ -0,0 +1,69 @@ +{ + "full_description": "place the container onto the plate", + "schema": "{A} notifies the plate to be placed onto, {B} notifies the arm to manipulate the container, {a} notifies the arm to catch the container", + "preference": "num of words should not exceed 10. Degree of detail avg 5", + "seen": [ + "Move the container with {B} to {A}.", + "Use {B} to place the container onto {A}.", + "Transfer the container and drop it on {A}.", + "Move and release the container onto {A}.", + "Grab the container with {a} and set on {A}.", + "Pick up the container with {a}, place it on {A}.", + "Secure the container, move it, and set it on {A}.", + "Carry the container and release it onto {A}.", + "Position the container using {B} onto {A}.", + "Lift the container with {B}, then place it onto {A}.", + "Lift container and drop it onto {A}.", + "Move the container to {A} and set it down.", + "Use {B} to grab the container and place it on {A}.", + "Catch the container, align it, place on {A}.", + "Secure container with {B}, move it, place on {A}.", + "Lift the container and position it on {A}.", + "Grab container using {B}, transfer it to {A}.", + "Place the container directly on {A}.", + "Catch {a}, move container over {A}, drop it.", + "Lift container toward {A} and carefully set it down.", + "Drop the container onto {A} using {B}", + "Deliver the container to {A} and release", + "Set the container down on {A} carefully", + "Place the container flat on {A} now", + "Move the container above {A} and drop it", + "Lift the container and set it onto {A}", + "Position {B} to lower the container on {A}", + "Move {B} and drop the container onto {A}", + "Use {B} to place the container into {A}", + "Guide the container with {B} onto {A}", + "Stick the container onto {A} using {B}.", + "Slide the container into position on {A}.", + "Place the container onto {A} with {B}.", + "Align the container and set onto {A}.", + "Grab the container using {a} and place it onto {A}.", + "Catch the container with {B} and drop onto {A}.", + "Lift the container and set it onto {A}.", + "Move the container and stick it onto {A}.", + "Position the container onto {A} using {B}.", + "Shift the container and place it on {A}.", + "Place the container onto {A}.", + "Catch the container and set it on {A}.", + "Drop the container onto {A}.", + "Use {a} to lower the container onto {A}.", + "Stick the container on top of {A}.", + "Grab the container and place on {A}.", + "Position the container and set on {A}.", + "Slide the container to {A} using {a}.", + "Place the container firmly onto {A}.", + "Hold the container with {a} and drop onto {A}." + ], + "unseen": [ + "Place the container onto {A}.", + "Set the container on top of {A}.", + "Catch {a}, move it over {A}, set it down.", + "Grab container with {B}, place it on {A}.", + "Move {B} to set the container on {A}", + "Place the container onto {A} with {B}", + "Set the container on {A} with {B}.", + "Move and drop the container onto {A}.", + "Set the container on {A}.", + "Lower the container onto {A} using {a}." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_dual_shoes.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_dual_shoes.json new file mode 100644 index 00000000..9b6810e4 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_dual_shoes.json @@ -0,0 +1,69 @@ +{ + "full_description": "use both arms to pick up the two shoes on the table and put them in the shoebox, with the shoe tip pointing to the left", + "schema": "{A} notifies one shoe(use 'two {A}' or 'a pair of {}').{B} nofity the shoebox", + "preference": "num of words should not exceed 10.IN EACH INSTRUCTION, YOU MUST STRESS THE SHOE TIP POINTING TO THE LEFT", + "seen": [ + "Put two {A}, tips to the left, into {B}.", + "Pick up two {A}, tips left, drop them into {B}.", + "Collect two {A}, tips facing left, and put in {B}.", + "Move two {A}, tip ends left, and set them in {B}.", + "Transfer two {A}, tips left-aligned, into {B}.", + "Use arms to lift two {A}, tips left, and set in {B}.", + "Retrieve two {A}, orienting tips left, and place in {B}.", + "Handle two {A}, tips directed left, and drop them into {B}.", + "Pick two {A}, ensure tips facing left, and position in {B}.", + "Grasp two {A}, tips aligned left, and place them in {B}.", + "Grab two {A}, tip left, place them in {B}.", + "Pick up two {A} and set them in {B}, tips left.", + "Use hands to grab two {A}, tips left, drop into {B}.", + "Move two {A} into {B}, ensure tips point left.", + "Grab a pair of {A}, tips left, put them into {B}.", + "Use both arms to place two {A} into {B}, tips left.", + "Pick up two {A}, tips left, and set them in {B}.", + "Place two {A} into {B}, ensure shoe tips face left.", + "With both arms, move two {A} into {B}, tips facing left.", + "Grab two {A}, tips left, place them carefully in {B}.", + "Pick up two {A} together and place in {B}, tips left.", + "Ensure two {A} are in {B}, tips facing left.", + "Grab both {A} and put them into {B}, tips left.", + "Set two {A} into the {B}, align tips to the left.", + "Hold two {A}, drop them in {B}, tips pointing left.", + "Place two {A} inside {B}, make tips face left.", + "Lift two {A}, position them in {B}, tips to the left.", + "Drop a pair of {A} into {B}, tips align left.", + "Use both arms to set two {A} in {B}, tips left.", + "Lay down two {A} in {B}, tips oriented left.", + "Set two {A} in {B} with tips left.", + "Hold two {A}, tips left, and place in {B}.", + "Pick up two {A}, ensure tips point left.", + "Put two {A} into {B}, shoe tips left.", + "Make sure tips point left, then place {A} in {B}.", + "Grab and place two {A} in {B}, tips left.", + "Pick two {A} up from table, tips left.", + "Stick both {A} in {B}, shoe tips pointing left.", + "Place two {A} in {B}, tip left when set.", + "Lift and put two {A} in {B}, tips left.", + "Set two {A} in the {B}, tips pointing left.", + "Place two {A} tips leftward into the {B}.", + "Lift two {A}, ensure left tips, drop in {B}.", + "Move a pair of {A} into {B}, left tips first.", + "Slide both {A} into {B} with tips pointing left.", + "Take two {A}, ensure tips left, place in {B}.", + "Position two {A} into {B}, shoe tips leftward.", + "Put two {A} in {B} with tips facing left.", + "Arrange the two {A}, tips left, inside {B}.", + "Grab both {A}, ensure tips left, place in {B}." + ], + "unseen": [ + "Lift two {A}, tips pointing left, into {B}.", + "Grab two {A}, point tips left, place in {B}.", + "Pick up two {A}, tips left, into {B}.", + "Use both arms to move two {A} to {B}, tips pointing left.", + "Grab two {A} and set them in {B}, tips left.", + "Put a pair of {A} in the {B}, tips pointing left.", + "Grab two {A} with both hands, tip left.", + "Place two {A} into {B}, tips left.", + "Pick up two {A}, tips left, into {B}.", + "Grab a pair of {A}, place left tips in {B}." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_empty_cup.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_empty_cup.json new file mode 100644 index 00000000..90dfad1e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_empty_cup.json @@ -0,0 +1,69 @@ +{ + "full_description": "use an arm to place the empty cup on the coaster", + "schema": "{A} notifies the empty cup, {B} notifies the coaster", + "preference": "num of words should not exceed 10", + "seen": [ + "Move {A} to {B}", + "Place {A} on top of {B}", + "Slide {A} onto {B}", + "Set {A} down on {B}", + "Use an arm to drop {A} on {B}", + "Stick {A} onto {B} using the arm", + "Position {A} carefully on {B}", + "Use arm to place {A} on {B}", + "Set {A} carefully onto {B}", + "Drop {A} firmly on {B} using arm", + "Use the arm to set {A} on {B}", + "Lift {A} and place it over {B}", + "Move {A} onto {B} with the arm", + "Align {A} and set it down on {B}", + "Use an arm to place {A} on {B}", + "Grab {A} and lower it onto {B}", + "Position {A} above and drop it on {B}", + "Using the arm, align {A} over {B}", + "Relocate {A} precisely onto {B} with arm", + "Carefully place {A} onto {B} without arm", + "Use the arm to put {A} onto {B}.", + "Pick {A} up and drop it on {B}.", + "Grab {A}, then place it onto {B}.", + "Move {A} to {B} using the arm.", + "Stick {A} onto {B} using the arm.", + "Drop {A} on top of {B}.", + "Slide {A} onto {B} with the arm.", + "Use the arm to slide {A} onto {B}.", + "Grab {A} and set it onto {B}.", + "Set {A} on {B} using the arm.", + "Move {A} to {B} and set it down.", + "Using the arm, drop {A} onto {B}.", + "Grab {A} and place it on {B}.", + "Set {A} securely on {B} using the arm.", + "Use the arm to position {A} on {B}.", + "Stick {A} onto {B} carefully.", + "Position {A} on {B} with the arm.", + "Direct {A} to {B} and let it rest.", + "Lift {A} and place it onto {B}.", + "Using the arm, slide {A} onto {B}.", + "Use an arm to drop {A} on {B}.", + "Lift {A} and stick it onto {B}.", + "Grab {A} and place it over {B}.", + "Move {A} using the arm onto {B}.", + "Slide {A} gently onto {B} via the arm.", + "Make sure {A} ends up on {B}.", + "Position {A} squarely on top of {B}.", + "Use the arm to position {A} onto {B}.", + "Lift {A} up and drop it on {B}.", + "Set {A} onto {B} using the arm." + ], + "unseen": [ + "Place {A} onto {B}", + "Use arm to set {A} on {B}", + "Place {A} carefully onto {B}", + "Pick {A} and position it on {B}", + "Place {A} onto {B}.", + "Set {A} on top of {B}.", + "Place {A} onto {B} gently.", + "Set {A} on top of {B}.", + "Set {A} down on {B}.", + "Place {A} on top of {B}." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_fan.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_fan.json new file mode 100644 index 00000000..57c3c6e1 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_fan.json @@ -0,0 +1,69 @@ +{ + "full_description": "grab the fan and place it on a colored mat, ", + "schema": "{A} notifies the fan,{B} notifies the color of the mat(YOU SHOULD SAY {B} mat, or {B} colored mat), {a} notifies the arm to grab the fan", + "preference": "num of words should not exceed 15", + "seen": [ + "Place {A} on the {B} mat after grabbing it with {a} and align it toward the robot.", + "Grab {A} with {a} and ensure it's positioned on the {B} mat facing the robot.", + "Grab {A} and position it on the {B} mat, ensuring it faces the robot.", + "Lift {A}, place it on the {B} mat, and ensure it's facing the robot.", + "Use {a} to pick {A}, set it on the {B} mat, and face it toward the robot.", + "Grab {A} and carefully place it on the {B} mat facing toward the robot.", + "Pick up {A} with {a}, place it on the {B} mat, and turn it toward the robot.", + "Lift {A} and set it on the {B} mat, ensuring it faces the robot.", + "Use {a} to grab {A}, then align it on the {B} mat facing the robot.", + "Pick {A}, place it on the {B} mat, and ensure it points toward the robot.", + "Use {a} to grab {A}, put it on the {B} mat, and face it toward the robot", + "Lift {A} with {a}, place it on the {B} mat, and point it at the robot", + "Set {A} on the {B} mat and make sure it faces the robot", + "With {a}, grab {A} and position it on the {B} mat facing the robot", + "Take {A}, place it on the {B} mat, ensure it points at the robot", + "Grab {A} with {a}, set it on the {B} mat, and align it to face the robot", + "Lift {A} and put it on the {B} mat so it faces the robot", + "Use {a} to pick {A}, set it on the {B} mat, and direct it toward the robot", + "Place {A} on the {B} mat and confirm it is pointing at the robot", + "Take {A} with {a}, put it on the {B} mat, and make it face the robot", + "Use {a} to pick up {A} and place it on {B} mat.", + "Pick up {A} and ensure it faces the robot on the {B} mat.", + "Set {A} onto the {B} colored mat, oriented towards the robot.", + "Grab {A} with {a}, making sure it faces the robot on the {B} mat.", + "Place {A} on the {B} mat and position it to face the robot.", + "Lift {A} using {a} and put it on the {B} mat facing the robot.", + "Position {A} on the {B} mat so it faces the robot.", + "Grab {A} with {a}, place it on the {B} mat, ensure it faces the robot.", + "Pick up {A} and place it on the {B} mat with it facing the robot.", + "Use {a} to grab {A}, set it on {B} mat, and make it face the robot.", + "Pick {A}, align it toward the robot, and drop it on the {B} mat.", + "With {a}, grab {A}, align it to face the robot, and put it on the {B} mat.", + "Pick up {A} and place it on the {B} mat ensuring it faces the robot.", + "Grab {A} using {a} and set it on the {B} colored mat, facing the robot.", + "Grab {A}, position it to face the robot, and place it on the {B} mat.", + "Pick {A} with {a}, ensure it faces the robot, and put it on the {B} mat.", + "Lift {A}, align it toward the robot, and position it on the {B} mat.", + "Using {a}, grab {A}, face it towards the robot, and set it on the {B} mat.", + "Take {A} and place it on the {B} mat, making sure it faces the robot.", + "Pick {A} with {a}, align it to face the robot, and set it on the {B} mat.", + "Place {A} on the {B} mat and ensure it faces the robot.", + "Using {a}, grab {A} and put it on the {B} mat facing the robot.", + "Set {A} on the {B} colored mat ensuring it faces the robot.", + "Grab {A} using {a} and place it on the {B} mat ensuring it faces the robot.", + "Place {A} on the {B} mat and verify it is facing the robot.", + "Pick {A} with {a} and set it on the {B} mat facing the robot.", + "Put {A} on the {B} mat and make sure it faces the robot.", + "Grab {A} using {a} and position it on the {B} mat facing the robot.", + "Place {A} on the {B} colored mat ensuring it faces the robot.", + "Using {a}, grab {A} and set it on the {B} mat facing the robot." + ], + "unseen": [ + "Pick up {A} and set it on the {B} mat facing the robot.", + "Use {a} to grab {A}, then place it on the {B} mat facing the robot.", + "Grab {A} and set it on the {B} mat facing the robot", + "Pick {A}, place it on the {B} mat, face it toward the robot", + "Grab {A} and set it on the {B} mat.", + "Place {A} onto the {B} colored mat facing the robot.", + "Grab {A} and set it on the {B} mat facing the robot.", + "Use {a} to grab {A} and place it on the {B} mat facing the robot.", + "Pick {A} and set it on the {B} mat facing the robot.", + "Grab {A} with {a} and position it on the {B} mat facing the robot." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_mouse_pad.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_mouse_pad.json new file mode 100644 index 00000000..553189d9 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_mouse_pad.json @@ -0,0 +1,69 @@ +{ + "full_description": "grab the mouse and place it on a colored mat", + "schema": "{A} notifies the mouse, {B} notifies the color of the mat(YOU SHOULD SAY {B} mat, or {B} colored mat), {a} notifies the arm to grab the mouse", + "preference": "num of words should not exceed 15", + "seen": [ + "Take the {A} and place it on the {B} mat.", + "Lift the {A} using {a} and move it to the {B} mat.", + "Use {a} to grab the {A} and set it on the {B} mat.", + "Pick up the {A} with {a} and place it on the {B} mat.", + "Grip the {A} and move it onto the {B} mat.", + "Hold the {A} and transfer it to the {B} mat.", + "Take hold of the {A} and place it on the {B} mat.", + "Lift the {A} and position it on the {B} mat.", + "Using {a}, grab the {A} and set it on the {B} mat.", + "Pick up the {A} with {a} and move it to the {B} mat.", + "Use {a} to grab {A} and drop it on the {B} colored mat.", + "Lift {A} with {a} and place it on the {B} mat.", + "Grab {A} and move it onto the {B} mat.", + "Use {a} to pick {A} and place it on the {B} mat.", + "Take {A} and put it on the {B} mat.", + "With {a}, grab {A} and position it on the {B} mat.", + "Pick up {A} and set it on the {B} colored mat.", + "Lift {A} using {a} and drop it on the {B} colored mat.", + "Take {A} and place it onto the {B} mat.", + "Pick {A} with {a} and move it to the {B} mat.", + "Take {A} and place it on the {B} colored mat.", + "Use {a} to grab {A} and position it onto the {B} mat.", + "Lift {A} and set it down on the {B} mat.", + "Grip {A} using {a} and move it to the {B} mat.", + "Hold {A} and carefully place it on the {B} mat.", + "Grab {A} with {a} and lay it onto the {B} colored mat.", + "Take {A} and adjust it to rest on the {B} mat.", + "Lift {A} up with {a} and stick it onto the {B} mat.", + "Grab {A} and place it firmly on the {B} mat.", + "Use {a} to pick {A} up and set it on the {B} colored mat.", + "Grab the {A} using {a} and place it on the {B} mat.", + "Use {a} to lift the {A} and set it on the {B} mat.", + "Grab the {A} and position it onto the {B} mat.", + "Pick the {A} and place it on the {B} colored mat.", + "Lift the {A} using {a} and stick it on the {B} mat.", + "Take hold of {A} with {a} and drop it on the {B} mat.", + "Slide the {A} onto the {B} mat gently.", + "Use {a} to grab the {A} and place it on the {B} colored mat.", + "Position the {A} on the {B} mat by picking it up.", + "Lift the {A} and put it down onto the {B} mat.", + "Lift {A} and place it onto {B} mat.", + "With {a}, grab {A} and position it on {B} mat.", + "Take {A} and put it onto {B} mat.", + "Using {a}, grab {A} and place it on {B} colored mat.", + "Pick up {A} and move it to {B} mat.", + "With {a}, pick up {A} and lay it on {B} mat.", + "Lift {A} and stick it on {B} mat.", + "Using {a}, pick {A} and position it on {B} colored mat.", + "Grab {A} and drop it on {B} mat.", + "With {a}, take {A} and set it on {B} mat." + ], + "unseen": [ + "Pick up the {A} and set it on the {B} mat.", + "Grab the {A} and position it on the {B} mat.", + "Grab {A} and set it on the {B} mat.", + "Pick {A} and position it on the {B} colored mat.", + "Grab {A} and set it on the {B} mat.", + "Pick up {A} with {a} and drop it onto the {B} mat.", + "Pick up the {A} and set it on the {B} mat.", + "Lift the {A} and drop it on the {B} mat.", + "Grab {A} and set it on {B} mat.", + "Use {a} to pick up {A} and drop it on {B} mat." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_object_basket.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_object_basket.json new file mode 100644 index 00000000..905599dd --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_object_basket.json @@ -0,0 +1,69 @@ +{ + "full_description": "use one arm to grab the target object and put it in the basket, then use the other arm to grab the basket, and finally move the basket slightly away", + "schema": "{A} notifies the target object, {B} notifies the basket, {a} notifies the arm to grab the target object. {b} notifies the arm to grab the basket", + "preference": "num of words should not exceed 10. Degree of detail avg is six.", + "seen": [ + "Use {a} to grab {A}, then drop it in {B}.", + "Use {a} to pick {A}, then use {b} for {B}.", + "Grab {A}, drop it in {B}, then move {B}.", + "Place {A} into {B} and push {B} slightly away.", + "Pick {A} using {a}, put it in {B}, and shift {B}.", + "Lift {A} using {a}, drop it in {B}, then push {B} via {b}.", + "Grab {A}, place it in {B}, then move {B} away.", + "Pick up {A}, put it in {B}, shift {B} a little.", + "Use {a} to grab {A}, place it in {B}, and move {B} using {b}.", + "Lift {A}, drop it in {B}, then slightly relocate {B}.", + "Use one arm to grab {A}.", + "Pick {A}, place it in {B}.", + "Grab {A}, set it into {B}.", + "Use the other arm to move {B}.", + "Pick {A}, put it inside {B}.", + "Grab {A} and drop it in {B}.", + "Use one arm to place {A} in {B}.", + "Pick and move {A}, then shift {B}.", + "Lift {A}, place it into {B}, move {B}.", + "Use one arm to grab {B} and move it.", + "Use {a} to put {A} in {B}.", + "Grab {A}, drop it in {B}, shift {B}.", + "Move {A} to {B}, then shift {B}.", + "Use {a} to place {A} into {B}.", + "Put {A} in {B} and pull {B} away.", + "Grab {A}, drop in {B}, and move {B}.", + "Lift {A} using {a}, put it in {B}.", + "Pick {A}, place it in {B}, shift {B}.", + "Use {a} to move {A} into {B}, shift {B}.", + "Put {A} in {B}, then move {B} away slightly.", + "Pick up {A} and set it inside {B}.", + "Move {A} using {a}, then place it in {B}.", + "Place {A} in {B}, then grab {B}.", + "Use {b} to grab {B} and move it slightly.", + "Grab {B} and shift it away.", + "Use {b} to pick up {B} and move it aside.", + "Pick up {A}, place it in {B}, grab {B}.", + "Grab {A} with {a}, place it in {B}, then grab {B}.", + "Use {a} to grab {A}, drop it in {B}, grab {B}.", + "Set {A} in {B}, and shift {B} away.", + "Pick up {A} and drop it in {B}, then move {B}.", + "Take {A}, set it in {B}, shift {B} lightly.", + "Use one arm to place {A} in {B}, adjust {B}.", + "Grab {A} with {a}, put it into {B}.", + "Pick {A} and position it in {B}, move {B} slightly.", + "Grab {A} with one arm, drop {A} in {B}.", + "Take {A}, put {A} into {B}, shift {B}.", + "Use one arm to grab {A}, place it in {B}, then move {B}.", + "Pick {A}, drop {A} in {B}, slide {B} lightly.", + "Grab {A} using {a}, drop {A} in {B}, then adjust {B}." + ], + "unseen": [ + "Grab {A} and put it into {B}.", + "Pick up {A}, place it in {B}, move {B}.", + "Grab {A} and place into {B}.", + "Move {A} to {B}, then shift {B}.", + "Pick up {A} and drop in {B}.", + "Place {A} in {B} and move it.", + "Grab {A} and put it in {B}.", + "Use {a} to grab {A} and place it in {B}.", + "Grab {A}, put it in {B}, move {B}.", + "Use one arm to grab {A}, place it in {B}." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_object_scale.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_object_scale.json new file mode 100644 index 00000000..43b1bed6 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_object_scale.json @@ -0,0 +1,69 @@ +{ + "full_description": "use one arm to grab the object and put it on the scale", + "schema": "{A} notifies the scale, {B} notifies the object, {a} notifies the arm to grab the object", + "preference": "num of words should not exceed 10. Degree of detail avg is six.", + "seen": [ + "Use {a} to grab {B} for {A}", + "Place {B} on {A} after picking it", + "Grab {B} with {a} and set it on {A}", + "Lift {B} with {a} and position it on {A}", + "Pick {B} and carefully move it to {A}", + "Grab {B}, then place it smoothly on {A}", + "Use {a} to hold {B} and drop it on {A}", + "Lift {B} and stick it on {A} carefully", + "Hold {B} with {a}, set it down on {A}", + "Pick {B} and lay it down gently onto {A}", + "Set {B} down on {A} using one arm", + "Place {B} on {A} with one arm", + "Put {B} on {A} without mention of arm", + "Lift {B} and place on {A}", + "Pick up {B} and set it on {A}", + "Use one arm to place {B} on {A}", + "Lift {B} with one arm and set on {A}", + "Set {B} onto {A} after grabbing", + "Grab {B} and drop it on {A}", + "Carry {B} with one arm to {A}", + "Pick up {B} and drop it on {A}.", + "Lift {B} with {a} and set it on {A}.", + "Place {B} onto {A} using {a}.", + "Move {B} to {A} and leave it there.", + "Pick {B} up and set it on {A}.", + "Use {a} to lift {B} and place on {A}.", + "Transport {B} to {A} and put it there.", + "Set {B} on {A} after grabbing it with {a}.", + "After grabbing {B}, place it on {A}.", + "Using {a}, grab {B} and transfer it to {A}.", + "Pick up {B} and set it on {A}", + "Using {a}, pick {B} then place it on {A}", + "Lift {B} and drop it onto {A}", + "With {a}, lift {B} to set it on {A}", + "Take {B} and place it over {A}", + "Using {a}, grab {B} and put it on {A}", + "Pick up {B} and drop it on {A}", + "With {a}, secure {B} then release it on {A}", + "Take {B} and lay it onto {A}", + "Using {a}, grasp {B} and position it on {A}", + "Use {a} to lift {B} onto {A}", + "Place {B} carefully on {A} using {a}", + "Lift {B} and position it on {A}", + "Grab {B} and carefully set it on {A}", + "Utilize {a} to grab {B} and place on {A}", + "Pick {B} with {a} and move it to {A}", + "Take {B} and put it down on {A}", + "Using {a}, grab {B} and place on {A}", + "Move {B} to the position on {A}", + "With {a}, pick {B} and position it on {A}" + ], + "unseen": [ + "Grab {B} and drop it on {A}", + "Pick {B} up and put it on {A}", + "Grab {B} and set it on {A}", + "Use one arm to pick {B} for {A}", + "Grab {B} and place it on {A}.", + "Use {a} to put {B} on {A}.", + "Grab {B} and place it on {A}", + "With {a}, grab {B} and position it on {A}", + "Grab {B} and put it on {A}", + "Pick up {B} and set it on {A}" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_object_stand.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_object_stand.json new file mode 100644 index 00000000..8a744d8e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_object_stand.json @@ -0,0 +1,69 @@ +{ + "full_description": "use appropriate arm to place the object on the stand", + "schema": "{A} notifies the object, {B} notifies the stand, {a} notifies the arm to grab the object", + "preference": "num of words should not exceed 10", + "seen": [ + "Grab {A} and set it on {B}", + "Pick {A} and position it on {B}", + "Move {A} using {a} and place on {B}", + "Set {A} on {B} using {a}", + "Grab and put {A} on {B}", + "Lift {A} and position on {B}", + "Position {A} on {B} with {a}", + "Pick {A} up and place on {B}", + "Grab {A} with {a} and move to {B}", + "Take {A} and set it on {B}", + "Use {a} to position {A} on {B}.", + "Move {A} onto {B}.", + "Grab {A} with {a} and place on {B}.", + "Set {A} in position on {B}.", + "Use {a} to move {A} onto {B}.", + "Place {A} on {B}.", + "Transfer {A} using {a} to {B}.", + "Move {A} to {B} using {a}.", + "Position {A} on {B}.", + "Place {A} precisely on {B}.", + "Grab {A} and set it onto {B}.", + "Set {A} in position on {B}.", + "Pick {A} with {a} and place on {B}.", + "Transfer {A} to {B} securely with {a}.", + "Move {A} to {B} and set it there.", + "Carefully place {A} onto {B}.", + "Lift {A} with {a} and position on {B}.", + "Grab and place {A} directly on {B}.", + "Pick up {A} and drop it on {B}.", + "Use {a} to lift {A} and set on {B}.", + "Pick up {A} with {a} and set it on {B}", + "Lift {A} and position it on {B}", + "Select {a}, grab {A}, and move it to {B}", + "Put {A} on {B} after picking it", + "Grab {A} using {a} and place it on {B}", + "Move {A} to {B} and release it", + "Use {a} to lift {A} and set it on {B}", + "Place {A} on {B} after grabbing it", + "With {a}, pick {A} and position it on {B}", + "Set {A} on {B} after moving it", + "Pick up {A} and set it on {B}.", + "Place {A} precisely on top of {B}.", + "Use {a} to grab {A} and place on {B}.", + "Lift {A} with {a} and align it on {B}.", + "Grab and move {A} to position it on {B}.", + "Locate {A}, pick it up, and place on {B}.", + "Pick up {A} using {a} and set it on {B}.", + "Take {A} with {a} and put it on {B}.", + "Pick {A} and place it carefully onto {B}.", + "Bring {A} to {B} and set it in place." + ], + "unseen": [ + "Use {a} to place {A} on {B}", + "Place {A} onto {B} with {a}", + "Place {A} on {B} with {a}.", + "Set {A} on {B}.", + "Use {a} to place {A} on {B}.", + "Place {A} on {B} using {a}.", + "Use {a} to grab {A} and place it on {B}", + "Grab {A}, then place it on {B}", + "Grab {A} using {a} and place on {B}.", + "Set {A} onto {B} using the right arm." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_phone_stand.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_phone_stand.json new file mode 100644 index 00000000..4a40eb63 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_phone_stand.json @@ -0,0 +1,21 @@ +{ + "full_description": "pick up the phone and put it on the phone stand", + "schema": "{A} notifies the phone, {B} notifies the phonestand. Arm use literal 'arm'", + "preference": "num of words should not exceed 5", + "seen": [ + "Lift {A} using arm.", + "Move {A} onto {B}.", + "Take {A} to {B}.", + "Hold {A} with arm.", + "Grab {A} and position.", + "Put {A} atop {B}.", + "Use arm to grab {A}.", + "Carry {A} to {B}.", + "Lift {A} onto {B}.", + "Place {A} using arm." + ], + "unseen": [ + "Pick up {A}.", + "Set {A} on {B}." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_shoe.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_shoe.json new file mode 100644 index 00000000..28db7208 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/place_shoe.json @@ -0,0 +1,69 @@ +{ + "full_description": "use one arm to grab the shoe from the table and place it on the mat", + "schema": "{A} notifies the shoe, {a} notifies the arm to manipulate the shoe", + "preference": "num of words should not exceed 15", + "seen": [ + "Use {a} to grab the {A} and put it on the mat", + "Take the {A} off the table and put it on the mat", + "Lift the {A} from the table using {a} and place it on the mat", + "Pick the {A} off the table and position it on the mat", + "Use {a} to lift the {A} from the table and set it on the mat", + "Grab the {A} on the table and move it to the mat", + "Take the {A} from the table using {a} and drop it on the mat", + "Lift the {A} from the table and place it onto the mat", + "Use {a} to grab the {A} from the table and set it on the mat", + "Pick the {A} from the table and place it gently on the mat", + "Take {A} from the table and place it on the mat", + "Use {a} to pick {A} off the table and move it onto the mat", + "With {a}, grab {A} from the table and place it on the mat", + "Lift {A} from the table and gently place it onto the mat", + "Use {a} to lift {A} from the table and set it on the mat", + "Take {A} from the table and carefully drop it on the mat", + "Grab {A} using {a}, move it from the table to the mat", + "Lift {A} using {a} from the table and place it on the mat", + "Pick {A} up from the table and drop it directly on the mat", + "Move {A} from the table to the mat in one fluid motion", + "Use {a} to grab {A} from the table and move it to the mat", + "Lift {A} from the table and place it carefully on the mat", + "Pick up {A} with {a} from the table and place it on the mat", + "Retrieve {A} from the table and set it on the mat", + "Use {a} to pick up {A} from the table and drop it on the mat", + "Take {A} from the table and put it on the mat", + "With {a}, grab {A} off the table and place it onto the mat", + "Move {A} from the table and place it down on the mat", + "Use {a} to lift {A} from the table and set it on the mat", + "Pick {A} up from the table and put it on the mat", + "Pick up {A} and move it to the mat", + "Grab {A} from the table with {a} and place it on the mat", + "Take {A} and put it on the mat", + "Use {a} to grab {A} and transfer it to the mat", + "Lift {A} and place it on the mat", + "Use {a} to pick {A} and position it on the mat", + "Pick up {A} from the table and set it on the mat", + "Grab {A} using {a} and drop it on the mat", + "Move {A} from the table to the mat", + "Take {A} with {a} and place it carefully on the mat", + "Move {A} from the table to the mat", + "Lift {A} off the table using {a} and drop it on the mat", + "Grab {A} from the table and place it on the mat", + "With {a}, pick {A} up from the table and put it on the mat", + "Take {A} from the table and position it on the mat", + "Using {a}, grasp {A} from the table and place it onto the mat", + "Remove {A} from the table and lay it on the mat", + "Pick up {A} with {a}, move it from the table, and set it on the mat", + "Transfer {A} from the table to the mat", + "Using {a}, lift {A} from the table and place it onto the mat" + ], + "unseen": [ + "Grab the {A} from the table and set it on the mat", + "Pick up the {A} from the table and place it on the mat", + "Pick up {A} from the table and set it on the mat", + "Grab {A} off the table and drop it on the mat", + "Grab {A} from the table and place it on the mat", + "Pick up {A} from the table and set it down on the mat", + "Grab {A} and set it on the mat", + "Use {a} to lift {A} and drop it on the mat", + "Pick up {A} from the table and set it on the mat", + "Use {a} to grab {A} from the table and place it on the mat" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/press_stapler.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/press_stapler.json new file mode 100644 index 00000000..de16dd3a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/press_stapler.json @@ -0,0 +1,69 @@ +{ + "full_description": "Use one arm to press the stapler.", + "schema": "{A} notifies the stapler, {a} notifies the arm to press the stapler", + "preference": "num of words should not exceed 15", + "seen": [ + "Push {A} using one arm", + "Push down on {A} to staple", + "Press {A} until it works", + "Apply pressure to {A} firmly", + "Push {A} down completely", + "Lower {A} using an arm", + "Use an arm to press {A}", + "Push the top of {A} down", + "Apply force to {A} with {a}", + "Press {A} firmly to staple", + "Press on {A} using {a}.", + "Push {A} down firmly.", + "Press {A} to complete the task.", + "Apply force to {A} to staple.", + "Use one arm to press {A}.", + "Push the stapler {A} with {a}.", + "Push {A} until it staples.", + "Push down on {A} firmly.", + "Apply pressure to {A} with {a}.", + "Firmly push down on {A}.", + "Firmly press {A} using an arm.", + "Push down on the stapler {A}.", + "Apply pressure on {A} until it works.", + "Press on {A} to activate it.", + "Use your {a} to press the stapler {A}.", + "Firmly push {A} using {a} to operate it.", + "Push the stapler {A} to activate it.", + "Press down on {A} until it functions.", + "Press the stapler {A} using an arm {a}.", + "Apply enough force to {A} to press it.", + "Place pressure on {A} to staple.", + "Use one arm {a} to press {A}.", + "Push down on {A} to staple papers.", + "Apply pressure to {A} using one arm {a}.", + "Press {A} to staple the sheets together.", + "Use your arm {a} to push {A} down.", + "Push down firmly on {A} to complete.", + "Apply one arm {a} to press {A} firmly.", + "Press {A} downward using your arm {a}.", + "Firmly press down on {A} to staple.", + "Push down on the stapler {A}.", + "Apply pressure to {A} with {a}.", + "Push the stapler {A} to operate it.", + "Lower {a} onto the stapler {A}.", + "Push down hard on {A} with {a}.", + "Press down on the stapler {A} firmly.", + "Operate {A} by pressing it with {a}.", + "Push down forcefully on the stapler {A}.", + "Engage {A} by using {a} to press it.", + "Simply press the stapler {A} downward." + ], + "unseen": [ + "Press down on {A} with {a}", + "Use {a} to press on {A}", + "Press down on {A} with {a}.", + "Use {a} to press {A}.", + "Press down on {A} firmly using {a}.", + "Use {a} to press the stapler {A}.", + "Push {A} down with one arm {a}.", + "Press {A} firmly using your arm.", + "Press the stapler {A} with the arm {a}.", + "Use {a} to press down on {A}." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/put_bottles_dustbin.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/put_bottles_dustbin.json new file mode 100644 index 00000000..87b1ce84 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/put_bottles_dustbin.json @@ -0,0 +1,69 @@ +{ + "full_description": "use arms to grab the bottles and put them into the dustbin to the left of the table", + "schema": "{A} notifies the first bottle, {B} notifies the second bottle, {C} notifies the third bottle, {D} notifies the dustbin", + "preference": "num of words should not exceed 20. Degree of detail avg is 6.", + "seen": [ + "Take {A}, {B}, and {C} one at a time and place them into {D}.", + "Grab {A} and {B}, put them into {D}, and repeat for {C}.", + "Use the arms to transfer {A}, {B}, and {C} into {D} one by one.", + "Move {A}, {B}, and {C} to {D} sequentially using the arms.", + "Drop {A}, {B}, and {C} into {D}, handling them one at a time.", + "With arms, place {A}, {B}, and {C} into {D} step by step.", + "Carefully grab {A}, {B}, and {C} and position them in {D}.", + "Using arms, pick up {A}, {B}, and {C}, dropping each into {D}.", + "Place {A}, {B}, and {C} into {D} after taking them individually.", + "Transfer {A}, {B}, and {C} into {D} using your arms, one by one.", + "Using arms, grab {A} and put it in {D}, repeat for {B} and {C}.", + "Take {A}, drop it in {D}, then move {B} and {C} to {D}.", + "Grab {A}, place it in {D}, then grab {B} and {C} for {D}.", + "Use arms to pick {A}, drop it in {D}, repeat for {B} and {C}.", + "Pick {A} and move it into {D}, repeat for {B} and {C}.", + "Using arms, grab {A} and {B}, then place them into {D} along with {C}.", + "Grab {A}, put it in {D}, follow by moving {B} and {C} there too.", + "Take {A} with arms, put it in {D}, repeat the same for {B} and {C}.", + "Pick {A}, drop it into {D}, then move {B} and {C} into {D}.", + "Using arms, grab {A} and place it in {D}, do the same for {B} and {C}.", + "Move {A}, {B}, and {C} into {D} one by one.", + "Grab each of {A}, {B}, and {C} and drop them into {D}.", + "Use the arms to move {A} into {D}, then repeat for {B} and {C}.", + "Place {A}, {B}, and {C} in {D} using the arms.", + "Transfer {A}, {B}, and {C} into {D} step by step.", + "Put {A} into {D}, followed by {B} and {C}.", + "Use arms to pick {A}, {B}, and {C}, dropping them into {D}.", + "Move {A} to {D}, then {B}, and finally {C}.", + "Grab {A}, {B}, and {C} sequentially, placing each into {D}.", + "Pick {A}, {B}, and {C} one at a time and put them into {D}.", + "Use arms to move {A}, {B}, and {C} into {D}.", + "Start with {A}, grab it and drop it into {D}, repeat for {B} and {C}.", + "Place {A}, {B}, and {C}, one at a time, into {D}.", + "Begin with {A}, move it to {D}, then continue with {B} and {C}.", + "Lift {A}, {B}, and {C}, dropping each into {D} sequentially.", + "Grab {A}, drop it in {D}, do the same for {B} and {C}.", + "Pick up {A}, {B}, and {C}, moving them one by one into {D}.", + "Use the arms to transfer {A}, {B}, and {C} into {D} step by step.", + "Move {A} into {D}, then do the same for {B} and {C}.", + "Carry {A} first, drop it into {D}, repeat with {B} and {C}.", + "Use arms to pick {A}, {B}, {C}, and place them one by one in {D}.", + "Transfer {A}, {B}, and {C} to {D} located to the table's left.", + "Grab {A}, {B}, and {C} and put them sequentially into {D}.", + "Use arms to grab {A}, {B}, {C}, and set them into {D} to the left.", + "Pick {A}, {B}, and {C} from the table and place them inside {D}.", + "Use arms to move {A}, {B}, and {C} to {D} on the table's left.", + "Place {A}, {B}, and {C} into {D} one after another.", + "Grab {A}, {B}, {C}, and drop them into {D} to the left of the table.", + "Use arms to pick {A}, {B}, and {C} and place them in {D}.", + "Take {A}, {B}, and {C} and place them into {D} on the left." + ], + "unseen": [ + "Pick up {A} and toss it into {D}; repeat for {B} and {C}.", + "Use arms to grab {A}, {B}, and {C} and drop them into {D}.", + "Pick {A}, drop it into {D}, repeat for {B} and {C}.", + "Grab {A} and place it in {D}, then do the same for {B} and {C}.", + "Pick up {A} and drop it into {D}, do the same for {B} and {C}.", + "Use arms to grab {A}, {B}, and {C}, placing each into {D}.", + "Pick up {A}, drop it into {D}, then repeat for {B} and {C}.", + "Take each bottle {A}, {B}, {C} one by one and place them in {D}.", + "Pick up {A}, {B}, and {C} one by one and drop them into {D}.", + "Move {A}, {B}, and {C} into {D} on the left of the table." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/put_object_cabinet.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/put_object_cabinet.json new file mode 100644 index 00000000..5f6cb182 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/put_object_cabinet.json @@ -0,0 +1,69 @@ +{ + "full_description": "use {a} to open the cabinet's drawer, and use another arm to put the object on the table to the drawer.", + "schema": "{A} notifies the object, {B} notifies the cabinet (use 'the drawer of {B}' or '{B}'s drawer'), {a} notifies the arm to open the drawer. Another arm to put the object into the drawer use literal 'the other arm'", + "preference": "num of words should not exceed 20. Degree of detail avg is 8.If {a} is mentioned in one description, 'the other arm' or 'another arm' must be mentioned in another description.Or otherwise both arm should not be mentioned. BUT {A} and {B} should always be mentioned in each description", + "seen": [ + "Open {B}'s drawer and place {A} inside it.", + "Access {B}'s drawer and set {A} inside.", + "Use {a} to open the drawer of {B} and then the other arm to place {A} inside.", + "Pull open {B}'s drawer with {a} and use the other arm to insert {A}.", + "Pull open {B}'s drawer and put {A} inside.", + "Open the drawer of {B} and place {A} into it.", + "Use {a} to open {B}'s drawer before the other arm places {A} inside.", + "With {a}, open {B}'s drawer and let the other arm drop {A} inside.", + "Open {B}'s drawer and move {A} into it.", + "Slide open {B}'s drawer and set {A} inside.", + "Open {B}'s drawer and transfer {A} into it.", + "Pull open the drawer of {B} and place {A} inside.", + "Use {a} to open the drawer of {B} and let the other arm set {A} inside.", + "Slide open {B}'s drawer with {a} and use the other arm to put {A} into it.", + "Open {B}'s drawer and carefully drop {A} into it.", + "Pull open the drawer of {B} and move {A} into it.", + "With {a}, open the drawer of {B} and place {A} inside using the other arm.", + "Use {a} to open {B}'s drawer, then transfer {A} into it with the other arm.", + "Open the drawer of {B} and set {A} inside.", + "Open {B}'s drawer and move {A} into it.", + "Use {a} to slide open {B}'s drawer, then the other arm to put {A} in.", + "Unlock the drawer of {B} and drop {A} into it.", + "Use {a} to open {B}'s drawer and use another arm to stick {A} inside.", + "Pull open {B}'s drawer and place {A} inside it.", + "Use {a} to open the drawer of {B}, then use the other arm to set {A} inside.", + "Slide out the drawer of {B} and stick {A} inside.", + "With {a}, open {B}'s drawer, then place {A} in using another arm.", + "Open the drawer of {B} and drop {A} inside.", + "Use {a} to pull {B}'s drawer open and use the other arm to stick {A} inside.", + "Slide open the drawer of {B} and set {A} into it.", + "Pull open {B}'s drawer and place {A} into it.", + "Open the drawer of {B} and move {A} into it.", + "Use {a} to slide open {B}'s drawer, then use the other arm to transfer {A} inside.", + "Unlock {B}'s drawer with {a} and set {A} inside using the other arm.", + "Slide open {B}'s drawer and drop {A} inside.", + "Pull open {B}'s drawer and place {A} inside.", + "Use {a} to pull the drawer of {B} open and the other arm to put {A} inside.", + "Open {B}'s drawer with {a} and place {A} inside using the other arm.", + "Open the drawer of {B} and move {A} into the drawer.", + "Pull {B}'s drawer open and transfer {A} into it.", + "Slide open the drawer of {B} before putting {A} inside.", + "Put {A} into {B}'s drawer once it is opened.", + "Use {a} to slide open {B}'s drawer and move {A} in with the other arm.", + "Place {A} inside the drawer of {B} after opening it using {a}.", + "Open the drawer of {B} before placing {A} inside it.", + "Set {A} into {B}'s drawer after opening it.", + "Use {a} to open {B}'s drawer and drop {A} inside with the other arm.", + "Slide open the drawer of {B} using {a} and place {A} into it with the other arm.", + "Open {B}'s drawer first, then place {A} inside.", + "Place {A} into the drawer of {B} after sliding it open." + ], + "unseen": [ + "Use {a} to open {B}'s drawer and the other arm to place {A} inside.", + "Open {B}'s drawer with {a} and use the other arm to put {A} inside.", + "Open the drawer of {B} with {a} and place {A} inside using the other arm.", + "Use {a} to pull open {B}'s drawer and the other arm to move {A} into it.", + "Open {B}'s drawer with {a} and place {A} inside using the other arm.", + "Pull out the drawer of {B} and set {A} into it.", + "Open the drawer of {B} with {a}, then the other arm places {A} inside.", + "Open {B}'s drawer and set {A} inside with the other arm.", + "Open {B}'s drawer using {a} and place {A} inside with the other arm.", + "Set {A} in the drawer of {B} after opening it with {a}." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/rotate_qrcode.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/rotate_qrcode.json new file mode 100644 index 00000000..4f77f65e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/rotate_qrcode.json @@ -0,0 +1,69 @@ +{ + "full_description": "Use arm to catch the qrcode board on the table, pick it up and rotate to let the qrcode face towards you", + "schema": "{A} notifies the qrcode board. {a} notifies the arm to pick the qrcode board", + "preference": "num of words should not exceed 15. Degree of detail avg is 6.", + "seen": [ + "Pick up {A} and rotate it so the QR code faces you", + "Use {a} to grab {A}, lift, and turn it QR code forward", + "Lift {A} from the table and rotate it towards you", + "Catch {A}, raise it, and turn it so the QR code faces you", + "Grab {A}, lift it from the table, and rotate it QR front", + "Use {a} to take {A} and turn it until the QR code faces you", + "Lift {A} from the surface and adjust its angle towards you", + "Employ {a} to seize {A}, raise it, and rotate it QR-forward", + "Take {A}, lift it, and orient it so the QR faces you", + "Use {a} to grab {A} and rotate it until the QR faces forward", + "Find {A}, grab it, and turn it towards yourself.", + "Use {a} to grab {A} and rotate the qrcode to face you.", + "Slide {A} off the table and turn it to face you.", + "Grab {A} with {a}, then rotate it to face yourself.", + "Locate {A}, pick it up, and adjust its angle.", + "Use {a} to lift {A} from the table and face the qrcode.", + "Catch {A}, lift it, and turn the qrcode towards you.", + "Grab {A} using {a}, then rotate it until the qrcode faces you.", + "Pick up {A} and adjust its position to face the qrcode towards you.", + "Lift {A} with {a}, then rotate it to make the qrcode visible.", + "Catch and lift {A}, then turn it to show the QR code.", + "Use {a} to grab {A} and rotate QR code towards you.", + "Grab {A} using {a}, lift, and rotate until QR code faces you.", + "Catch {A} with {a}, then rotate it to make the QR code visible.", + "Lift {A} from the table and rotate it so the code faces you.", + "Using {a}, catch {A} and rotate it to face the QR code.", + "Catch {A} using {a}, pick it up, and turn it to face the QR code.", + "Lift {A} and rotate it until the QR code faces you.", + "Use {a} to grab {A}, rotate, and face the QR code towards you.", + "Catch {A}, pick it up, and rotate to show the QR code.", + "Catch {A}, lift it, and rotate it QR code facing.", + "Use {a} to grab {A} and point its QR code toward you.", + "Lift {A} from the table, turning it QR code forward.", + "Take {A} from the table, rotating it QR code toward you.", + "Use {a} to lift {A} and rotate it QR code toward you.", + "Pick {A} up and turn its QR code toward you using {a}.", + "Catch {A}, lift it, and adjust its QR code to face you.", + "Grab {A} using {a}, then rotate the QR code to face forward.", + "Lift {A} and orient its QR code toward you with {a}.", + "Pick {A} up, rotate it, and ensure the QR code faces you.", + "Lift {A} from the table and turn it to face you.", + "Catch {A}, pick it up, and rotate to view the qrcode.", + "Take {A}, raise it, and make the qrcode face you.", + "Use {a} to pick {A} and turn it towards you.", + "Lift {A} and rotate until its qrcode faces you.", + "Catch {A} off the table and rotate its qrcode to you.", + "Pick {A} up, then rotate to make its qrcode visible.", + "Grab {A}, pick it up, and turn its qrcode toward you.", + "Lift {A} and rotate for its qrcode to face you.", + "Catch {A}, lift, and rotate to align the qrcode to you." + ], + "unseen": [ + "Catch {A} from the table and rotate it", + "Grab {A}, lift it, and turn it to face you", + "Catch {A} from the table and make it face you.", + "Pick {A} off the table using {a} and rotate it.", + "Pick {A} up from the table and rotate it.", + "Grab {A}, lift it, and rotate until the QR code faces you.", + "Catch {A} on the table and pick it up.", + "Pick up {A} and rotate it to face its QR code toward you.", + "Pick up {A} and rotate it facing you.", + "Grab {A}, lift it, and rotate to see the qrcode." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/scan_object.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/scan_object.json new file mode 100644 index 00000000..cd16142e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/scan_object.json @@ -0,0 +1,69 @@ +{ + "full_description": "Use one arm to pick the scanner and use the other arm to pick the object, and use the scanner to scan the object", + "schema": "{A} notifies the object, {B} notifies the scanner, {a} notifies the arm to pick the object, {b} notifies the arm to pick the scanner", + "preference": "num of words should not exceed 20. Degree of detail avg is 6.", + "seen": [ + "{b} picks up {B}, {a} holds {A}, and you scan {A} with {B}.", + "Use {b} to pick {B}, {a} to grab {A}, and scan {A} using {B}.", + "Pick {B}, grab {A}, and scan it with {B}.", + "{b} grabs {B}, {a} picks {A}, and {B} scans {A}.", + "{B} is picked with {b}, {A} grabbed with {a}, scan {A} using {B}.", + "Pick {A}, grab {B}, and use {B} to scan the object.", + "{b} holds {B}, {a} grabs {A}, and scans {A} with {B}.", + "Pick {B}, grab {A}, then scan {A} using {B}.", + "Grab {A}, pick {B}, and use {B} to scan {A}.", + "{a} picks {A}, {b} grabs {B}, and scans {A} with {B}.", + "Use one arm for {B} and the other for {A}, then scan {A} with {B}", + "Take {B}, grab {A}, and scan {A} using {B}", + "Hold {B} in one arm, pick up {A} with the other, then use {B} to scan {A}", + "Pick {B}, then pick {A}, and use {B} to scan {A}", + "Grab {B} using one arm, pick {A} using the other arm, then scan {A} with {B}", + "Take {B}, hold {A}, and use {B} to scan it", + "Use one arm to grab {B}, the other to pick {A}, and scan {A} with {B}", + "Pick {B}, grab {A}, and scan it with {B}", + "Grab {B} in one hand, pick {A} in the other hand, then scan {A} with {B}", + "Hold {B}, pick {A}, and scan {A} using {B}", + "Using {b}, pick {B}, then use {a} to grab {A} and scan it.", + "Pick {B} with {b}, grab {A} with {a}, and scan {A}.", + "Use one arm to grab {B}, the other to hold {A}, and scan it.", + "Take {B}, then grab {A}, and scan {A} with {B}.", + "Hold {B}, grab {A}, then scan {A} using {B}.", + "With {b}, pick {B}, then grab {A} with {a}, and scan {A}.", + "Take hold of {B} using {b}, grab {A} using {a}, and scan it.", + "Grab {B}, pick {A}, and perform a scan on {A} with {B}.", + "Use {b} to take {B}, then grab {A} with {a}, and scan it.", + "First, grab {B}, then pick {A}, and scan {A} using {B}.", + "Hold {B} with {b} and {A} with {a}, then scan {A} using {B}.", + "Pick {B}, grab {A}, and scan {A} with {B}.", + "Use {b} to take {B}, use {a} to pick {A}, and scan {A} using {B}.", + "Pick up {B}, hold {A}, and use {B} to scan {A}.", + "Hold {A} with one hand, grab {B} with the other, and scan {A}.", + "Take {B} using {b}, pick {A} using {a}, and scan {A} with {B}.", + "Pick {A}, grab {B}, and use the scanner to scan the object.", + "Take {B} in {b}, pick {A} in {a}, and scan {A} using {B}.", + "Lift up {B}, hold {A}, and scan {A} using the scanner.", + "Use {b} to pick {B}, use {a} to grab {A}, and scan {A} with {B}.", + "Using one arm, grab {B}, then grab {A} with the other arm, and scan.", + "Take {B} in one hand, {A} in the other, and perform the scan.", + "Hold {B} and {A} with different arms, then scan {A} using {B}.", + "First pick {B}, then {A}, and use {B} to scan {A}.", + "Grab {B} with one arm, {A} with the other, and use {B} to scan.", + "Pick up {B} in one arm and {A} in the other, then scan {A}.", + "Handle {B} and {A} separately, ensuring {B} scans {A}.", + "First grab {B}, then {A}, and use {B} to scan {A}.", + "Using one hand for {B} and the other for {A}, complete the scan.", + "Hold {B} in one arm and {A} in the other, then scan {A} with {B}." + ], + "unseen": [ + "Pick {B}, grab {A}, and scan {A} with {B}.", + "Grab {B}, then pick up {A}, and use {B} to scan {A}.", + "Grab {B} with one arm, take {A} with the other, then scan {A} using {B}", + "Pick up {B}, hold {A}, and scan it with {B}", + "Pick {B}, grab {A}, and scan it with {B}.", + "Grab {B}, take {A}, and scan {A} using {B}.", + "Grab {B} in one hand and {A} in the other, then scan {A} with {B}.", + "Take {B}, pick {A}, and use the scanner to perform the scan.", + "Lift {B} in one hand and grab {A} with the other, then scan it.", + "Pick up {B}, then pick {A}, and scan {A} using {B}." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/shake_bottle.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/shake_bottle.json new file mode 100644 index 00000000..ee8a9c8c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/shake_bottle.json @@ -0,0 +1,69 @@ +{ + "full_description": "Shake the bottle with proper arm", + "schema": "{A} notifies the bottle, {a} notifies the arm to pick the bottle", + "preference": "num of words should not exceed 10. Degree of detail avg is 6.", + "seen": [ + "Use {a} to grab {A} and shake it.", + "Grab {A} and give it a shake.", + "Pick {A} up using {a} and shake it.", + "Lift {A} and shake it thoroughly.", + "Grasp {A} with {a} and shake it.", + "Secure {A} and perform a shake.", + "Utilize {a} to hold {A} and shake it.", + "Hold {A} steady, then shake it.", + "Take {A} using {a} and give it a shake.", + "Catch {A} and shake it instantly.", + "Shake {A} after grabbing it.", + "Grab {A} with {a} and shake.", + "Hold {A}, shake it briefly.", + "Grip {A} using {a} and shake.", + "Lift {A} and shake it gently.", + "Shake {A} firmly after grabbing.", + "Pick {A} up with {a} and shake.", + "Use {a} to lift and shake {A}.", + "Grab {A} and shake it steadily.", + "Hold {A} with {a} and shake lightly.", + "Shake {A} thoroughly after grabbing it.", + "Use {a} to pick and shake {A}.", + "Lift {A} with {a} and shake it.", + "Grab {A} and move it to shake.", + "Shake the {A} properly after lifting.", + "Grab and shake {A} using {a}.", + "Pick {A}, shake it with {a}.", + "Shake {A} after holding it firmly.", + "Use {a} to lift and shake {A}.", + "Hold and shake the {A} carefully.", + "Use {a} to grab {A} and shake properly.", + "Shake {A} after grabbing it.", + "Lift {A} with {a} and give it a shake.", + "Grab {A} firmly, shake it, and put it down.", + "Using {a}, shake {A} and then place it back.", + "Pick up {A} and shake it well.", + "With {a}, pick {A} and shake properly.", + "Shake {A} after lifting it.", + "Use {a} to hold {A} and shake it.", + "Secure {A}, shake it, and place it down.", + "Pick up {A} and shake it.", + "Shake {A} after grabbing it.", + "Grab {A} using {a}, then shake.", + "Lift {A} with {a} and shake.", + "Hold {A} and shake it properly.", + "Shake {A} using {a} after grabbing.", + "Pick {A} and perform shaking motion.", + "Use {a} to lift and shake {A}.", + "Grab {A} carefully and shake.", + "Lift up {A} using {a}, then shake." + ], + "unseen": [ + "Shake {A} after picking it with {a}.", + "Pick up {A} and shake it.", + "Pick up {A} and shake it.", + "Use {a} to grab {A} and shake.", + "Pick and shake the {A} with {a}.", + "Grab {A} and give it a shake.", + "Pick up {A} with {a} and shake.", + "Grab {A}, shake it properly, then set it down.", + "Grab {A} and shake it.", + "Use {a} to pick {A}." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/shake_bottle_horizontally.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/shake_bottle_horizontally.json new file mode 100644 index 00000000..b2e9ee57 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/shake_bottle_horizontally.json @@ -0,0 +1,69 @@ +{ + "full_description": "Shake the bottle horizontally with proper arm", + "schema": "{A} notifies the bottle, {a} notifies the arm to pick the bottle", + "preference": "num of words should not exceed 10. Degree of detail avg is 6.", + "seen": [ + "Pick {A} using {a} and move it horizontally.", + "Lift {A} and shake it horizontally.", + "Use {a} to hold {A} and shake it sideways.", + "Grab {A} and shake in a horizontal motion.", + "Hold {A} with {a} and move it left and right.", + "Pick {A}, then shake it horizontally.", + "Lift {A} with {a}, then shake it side to side.", + "Grip {A}, then shake it back and forth.", + "Use {a} to pick {A} and shake it horizontally.", + "Hold {A} and shake it side to side.", + "Pick up {A} using {a} and shake sideways.", + "Shake {A} side-to-side after grabbing it.", + "Use {a} to grab {A} and shake horizontally.", + "Grab {A} and move it side-to-side repeatedly.", + "Secure {A} with {a}, shake in horizontal motion.", + "Hold {A} steady and shake it horizontally.", + "Take {A} in {a} and shake it back and forth.", + "Move {A} side-to-side after grabbing it.", + "Using {a}, grab {A} and shake it sideways.", + "Grab {A}, shake it horizontally, then release.", + "Shake {A} horizontally without mentioning {a}.", + "Grab {A} using {a} and move it side-to-side.", + "Pick up {A} and shake it horizontally.", + "Hold {A} with {a} and shake horizontally.", + "Shake {A} smoothly without using {a} reference.", + "Utilize {a} to grab {A} and shake sideways.", + "Simply shake {A} horizontally without {a} details.", + "Take hold of {A} using {a} and move horizontally.", + "Grab and shake {A} horizontally without mentioning {a}.", + "Use {a} to hold {A} firmly and shake horizontally.", + "Hold {A} and move it side to side.", + "Grab {A} with {a} and shake horizontally.", + "Pick {A} up and shake it horizontally.", + "Lift {A} using {a} and shake it sideways.", + "Shake {A} from side to side.", + "Use {a} to grab {A} and move it horizontally.", + "Pick up {A} and shake it side to side.", + "Hold {A} using {a} and shake it horizontally.", + "Lift {A} and move it back and forth.", + "With {a}, grab {A} and shake it horizontally.", + "Pick up {A} with {a}, shake it sideways.", + "Using {a}, shake {A} horizontally.", + "Lift {A} and move it side-to-side.", + "Shake {A} horizontally after lifting with {a}.", + "Pick up {A} and shake it from side to side.", + "Using {a}, pick up {A} and shake sideways.", + "Shake {A} side-to-side after grabbing it.", + "Lift {A} using {a} and shake horizontally.", + "Hold {A} and move it side to side.", + "Pick up {A} using {a}, shake it horizontally." + ], + "unseen": [ + "Grab {A} with {a} and shake horizontally.", + "Shake {A} side-to-side after picking it up.", + "Grab {A} with {a}, shake horizontally.", + "Shake {A} horizontally after grabbing it.", + "Grip {A} and shake it horizontally.", + "Use {a} to hold {A} and shake sideways.", + "Grab {A} and shake it horizontally.", + "Use {a} to pick {A} and shake it.", + "Shake {A} horizontally after grabbing.", + "Grab {A}, shake it horizontally." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/stack_blocks_three.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/stack_blocks_three.json new file mode 100644 index 00000000..e1074304 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/stack_blocks_three.json @@ -0,0 +1,69 @@ +{ + "full_description": "there are three blocks on the table, the color of the blocks is , , and ", + "schema": "{A} notifies the red block, {B} notifies the green block, {C} notifies the blue block, {a} notifies the arm to manipulate the red block, {b} notifies the arm to manipulate the green block, {c} notifies the arm to manipulate the blue block", + "preference": "num of words should not exceed 20. Degree of detail avg 8", + "seen": [ + "Shift {A}, {B}, {C} to the table's center, then stack {C} on {B}, and {B} on {A}.", + "Stack {C} over {B} and {B} over {A} after moving all blocks to the center.", + "Use {a}, {b}, {c} to place {A}, {B}, {C} at the center and stack them accordingly.", + "Grab {A}, {B}, and {C} using {a}, {b}, {c}, move them to the center, then stack them.", + "Move {A}, {B}, and {C} to the center using {a}, {b}, {c}, and stack them with {C} on top.", + "Use {a}, {b}, and {c} to center {A}, {B}, and {C}, then stack {C} above {B} and {B} above {A}.", + "Relocate {A}, {B}, and {C} to the center and stack {C} on {B} and {B} on {A}.", + "Reposition {A}, {B}, and {C} to the middle and arrange {C} above {B} and {B} above {A}.", + "Center {A}, {B}, and {C}, then stack them with {C} on {B} and {B} on {A}.", + "Place {A}, {B}, and {C} at the center and stack {C} on {B}, then {B} on {A}.", + "Place {A}, {B}, and {C} at the table's center; stack {C} over {B}, then {B} over {A}.", + "Use {a}, {b}, and {c} to move {A}, {B}, {C} to the center and stack {C} on {B}, {B} on {A}.", + "With {a}, {b}, and {c}, shift {A}, {B}, and {C} to the center and arrange {C} over {B}, {B} on {A}.", + "Use arms {a}, {b}, and {c} to centralize {A}, {B}, {C} and stack {C} above {B}, then {B} above {A}.", + "Centralize {A}, {B}, and {C} before stacking {C} on {B} and {B} on {A}.", + "Move {A}, {B}, and {C} to the middle first, then stack {C} on {B} and {B} on {A}.", + "Arrange {A}, {B}, and {C} in the table's center and stack {C} atop {B}, then {B} atop {A}.", + "With {a}, {b}, {c}, position {A}, {B}, {C} at the table's center and stack {C} on {B}, {B} on {A}.", + "Using {a}, {b}, {c}, place {A}, {B}, {C} centrally and stack {C} atop {B}, then {B} atop {A}.", + "Position {A}, {B}, and {C} in the center and stack {C} on {B}, followed by {B} on {A}.", + "Bring {A}, {B}, and {C} to the center and stack {B} over {A}, {C} over {B}.", + "Use {a}, {b}, and {c} to move {A}, {B}, and {C} to the center, then stack {C} on {B} and {B} on {A}.", + "Relocate {A}, {B}, and {C} to the center with {a}, {b}, {c}, and stack {C} on {B}, {B} on {A}.", + "Shift {A}, {B}, and {C} to the center using {a}, {b}, {c}, then pile {C} on {B}, {B} on {A}.", + "Move {A}, {B}, and {C} to the center and stack {B} on {A}, {C} on {B}.", + "Bring {A}, {B}, and {C} to the table's center and arrange them by stacking {C} over {B} and {B} over {A}.", + "Place {A}, {B}, {C} in the middle and stack them using {a}, {b}, {c}, {B} on {A}, {C} on {B}.", + "Adjust {A}, {B}, {C} to the center and use {a}, {b}, {c} to stack {C} on {B}, {B} on {A}.", + "Reposition {A}, {B}, and {C} to the center, stacking {B} on {A} and {C} on {B}.", + "With {a}, {b}, {c}, move {A}, {B}, {C} to the center and stack {B} on {A}, {C} on {B}.", + "Place {A}, {B}, and {C} at the center, then stack {C} onto {B} and {B} onto {A}.", + "Gather {A}, {B}, and {C} at the table's center and stack {C} on {B}, then {B} on {A}.", + "Move {A}, {B}, and {C} to the center of the table using {a}, {b}, and {c}, then stack them.", + "Using {a}, {b}, and {c}, bring {A}, {B}, and {C} to the center and stack {C} on {B}, {B} on {A}.", + "Transfer {A}, {B}, and {C} to the center with {a}, {b}, and {c}, stacking {C} on {B} and {B} on {A}.", + "Bring {A}, {B}, and {C} to the center point and arrange them by stacking {C} atop {B} and {B} atop {A}.", + "Relocate {A}, {B}, and {C} to the table's center, stacking {C} over {B} and {B} over {A}.", + "Move {A}, {B}, and {C} to the middle and position {C} on {B}, {B} on top of {A}.", + "Place {A}, {B}, and {C} at the center, using {a}, {b}, and {c} to stack {C} on {B} and {B} on {A}.", + "Transfer {A}, {B}, and {C} to the center, arranging {C} on top of {B} and {B} on {A} with {a}, {b}, {c}.", + "Position {A}, {B}, and {C} centrally. Place {B} on {A}, then set {C} on {B}.", + "Move {A}, {B}, and {C} to the center. Stack {C} on {B} and {B} on {A}.", + "Bring {A}, {B}, and {C} to the middle. Stack {B} onto {A} and {C} onto {B}.", + "Use {a}, {b}, and {c} to move {A}, {B}, and {C} to the center and stack them.", + "Bring {A}, {B}, and {C} to the center using {a}, {b}, and {c}. Stack {B} on {A}.", + "Use {a}, {b}, and {c} to place {A}, {B}, and {C} in the center. Stack {C} on top.", + "With {a}, {b}, and {c}, move {A}, {B}, and {C} centrally and stack {B} on {A}.", + "Use {a}, {b}, and {c} to centralize {A}, {B}, and {C} and build a stack with them.", + "Place {A}, {B}, and {C} in the center, then arrange {B} on {A} and {C} on {B}.", + "Move {A}, {B}, and {C} to the table's center and stack {B} over {A}, {C} over {B}." + ], + "unseen": [ + "Move {A}, {B}, and {C} to the table's center and stack them.", + "Transfer {A}, {B}, and {C} to the middle, then stack {C} over {B} and {B} over {A}.", + "Move {A}, {B}, and {C} to the center, then stack {C} on {B} and {B} on {A}.", + "Bring {A}, {B}, {C} to the table's center and stack them: {C} on {B}, {B} on {A}.", + "Place {A}, {B}, and {C} at the table's center, then stack {C} on {B} and {B} on {A}.", + "Move {A}, {B}, and {C} to the center, then stack {C} on {B}, and {B} on {A}.", + "Move {A}, {B}, and {C} to the center of the table, then stack {C} on {B} and {B} on {A}.", + "Bring {A}, {B}, and {C} to the center, stacking {C} on {B} and {B} on {A}.", + "Bring {A}, {B}, and {C} to the table's center. Stack {B} on {A} and {C} on {B}.", + "Move {A}, {B}, and {C} to the center, then stack {B} over {A} and {C} over {B}." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/stack_blocks_two.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/stack_blocks_two.json new file mode 100644 index 00000000..eab615f0 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/stack_blocks_two.json @@ -0,0 +1,69 @@ +{ + "full_description": "there are two blocks on the table, the color of the blocks is , , and ", + "schema": "{A} notifies the red block, {B} notifies the green block, {a} notifies the arm to manipulate the red block, {b} notifies the arm to manipulate the green block", + "preference": "num of words should not exceed 20. Degree of detail avg 8", + "seen": [ + "{a} grabs {A}, places it in the center, {b} stacks {B} on {A}.", + "Shift {A} and {B} to the center and stack {B} on {A}.", + "Move {A} to the center, then {b} stacks {B} on {A}.", + "{a} relocates {A} and {b} moves {B}, then stacks {B} on {A}.", + "Set {A} and {B} at the center, stacking {B} atop {A}.", + "Place {A} and {B} centrally, then stack {B} on {A}.", + "Move {A} and {B} to the center, then stack {B} above {A}.", + "{a} takes {A}, sets it in the center, {b} places {B} on {A}.", + "Relocate {A} and {B} to the center, then layer {B} over {A}.", + "{a} moves {A} to the center, {b} stacks {B} onto {A}.", + "Shift {A} and {B} to the middle, then stack {B} over {A}.", + "Grab {A} using {a}, position it at the center, and do the same for {B} using {b}, stacking {B} on {A}.", + "Pick up {A} with {a} and {B} with {b}, move both to the table's center and stack {B} over {A}.", + "Use {a} to move {A} and {b} to move {B} to the center, then place {B} onto {A}.", + "Relocate {A} and {B} to the table's midpoint, and stack {B} on top of {A}.", + "Pick up {A} and {B}, set them at the center, then position {B} above {A}.", + "Place {A} in the center, then move {B} there and stack {B} on {A}.", + "Utilize {a} to grab {A} and {b} to grab {B}, move them to the middle, and stack {B} above {A}.", + "Set {A} at the center, then stack {B} over {A} after placing it alongside.", + "Move {A} and {B} to the center, using {a} and {b}, and stack {B} on top of {A}.", + "Shift {A} and {B} to the center, then position {B} on {A}.", + "Use {a} to move {A}, then {b} to stack {B} on {A} at the center.", + "Place {A} and {B} at the center and then put {B} on {A}.", + "Move {A} using {a}, stack {B} on {A} after centering them.", + "Center {A} and {B} on the table and stack {B} on top of {A}.", + "Use {b} to center {B}, place it on {A} after moving them to the center.", + "Set {A} and {B} at the table's center, then stack {B} on {A}.", + "Move {A} to the center using {a}, then stack {B} on {A} using {b}.", + "Place {A} and {B} together at the center, then stack {B} on top.", + "Use {a} for {A}, then stack {B} on {A} after placing both at the center.", + "Use {a} to move {A} and {b} to stack {B}.", + "Bring {A} to the center, then stack {B} over it.", + "Relocate {A} to the middle and place {B} on it.", + "Move {A} with {a}, then stack {B} on it using {b}.", + "Position {A} at the center and stack {B} above it.", + "Use {a} to center {A}, then stack {B} with {b}.", + "Shift {A} to the center and place {B} on top.", + "Use {a} to move {A} and {b} to position {B}.", + "Move {A} to the middle and put {B} on top of it.", + "Center {A} using {a}, then stack {B} with {b} on top.", + "Move {A} and {B} to the center; stack {B} on top of {A}.", + "Grab {A}, set it in the center, then stack {B} over it.", + "Use {a} for {A}, place it in the center, then use {b} to stack {B} above {A}.", + "Position {A} and {B} at the center, then stack {B} on {A}.", + "Move {A} and put it in the center; stack {B} on top using {b}.", + "Set {A} in the center, then position {B} on top of it.", + "Use {a} to place {A} in the center; stack {B} on it with {b}.", + "Put {A} in the center, then add {B} above it.", + "Move {A} to the middle, use {b} to stack {B} right on top.", + "Place {A} and {B} in the center, then stack {B} above {A}." + ], + "unseen": [ + "Move {A} and {B} to the center, then stack {B} on {A}.", + "Bring {A} and {B} to the center, then position {B} above {A}.", + "Take {A} and {B}, move them to the table's center, then stack {B} on {A}.", + "Move {A} and {B} to the center and place {B} atop {A}.", + "Move {A} and {B} to the table's center, then stack {B} on {A}.", + "Grab {A}, place it at the center, then stack {B} on {A}.", + "Place {A} in the middle, then stack {B} on it.", + "Move {A} to the center, then put {B} on top.", + "Place {A} in the center, then stack {B} on it.", + "Use {a} to move {A} to the center, then set {b} and stack {B} on {A}." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/stack_bowls_three.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/stack_bowls_three.json new file mode 100644 index 00000000..28ca77ee --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/stack_bowls_three.json @@ -0,0 +1,69 @@ +{ + "full_description": "stack the three bowls on top of each other", + "schema": "{A} notifies the bowls", + "preference": "num of words should not exceed 15. Degree of detail avg 5", + "seen": [ + "Place the bowls {A} in a stacking order.", + "Arrange {A} with the largest bowl at the base.", + "Pick up {A} and stack them from biggest to smallest.", + "Align bowls {A} in descending size, forming a stack.", + "Position {A} one on top of the other largest to smallest.", + "Grab {A}, stack from largest at the base to the smallest.", + "Form a tower with bowls {A}, placing smallest last.", + "Organize bowls {A} by stacking them in size order.", + "Stack {A}, starting with the largest bowl at the bottom.", + "Place {A} in a stack with the largest at the foundation.", + "Place each {A} on top of the previous one.", + "Grab {A}, arrange them vertically.", + "Lift each {A} and stack them together neatly.", + "Take {A}, stack them from bottom to top.", + "Arrange {A} one on top of another.", + "Stack {A} from the base up to the top.", + "Carefully stack {A} piece by piece.", + "Hold {A}, place them one after another.", + "Start with the bottom {A} and finish stacking.", + "Stack {A} by arranging them sequentially.", + "Place the bottom {A}, then add the other two on top.", + "Arrange all three {A} into a vertical stack.", + "Grab a {A}, set it down, and stack the rest on it.", + "Place the first {A} down, then stack the next two above it.", + "Start with one {A}, and stack the other two on top.", + "Arrange the three {A} by stacking them one over the other.", + "Take the bottom {A} and sequentially pile the other two above.", + "Pick each {A} and stack them in order one on top of another.", + "Place one {A} down and carefully stack the others above it.", + "Position the first {A}, and sequentially add the two above it.", + "Place three {A} on top of each other.", + "Use arms to stack each {A} carefully.", + "Stack the {A} sequentially using your arms.", + "Stack all three {A} directly on top of each other.", + "Lift each {A} and stack it over the previous one.", + "Sequentially stack {A} using arms to align them.", + "Take each {A} and place them one by one on top.", + "Align and stack the {A} step by step using arms.", + "Stack the {A} without arms in a vertical sequence.", + "Use arms to place each {A} atop the previous one.", + "Place the first {A} down, then stack the remaining two.", + "Position the second {A} over the first, then the third on top.", + "Pick up each {A} and set them one over another.", + "Stack all three {A} by placing one on top of another.", + "Grab the first {A}, position the second, then add the third.", + "Use the arm to stack the {A} one by one.", + "Place the first {A}, then align the others on top using the arm.", + "Start with the first {A}, adding the next two in sequence.", + "Stack all {A} sequentially while handling one at a time.", + "Align the second {A} atop the first, then add the third above." + ], + "unseen": [ + "Stack {A} by placing smallest on top.", + "Put {A} in a pile, largest to smallest.", + "Stack {A} together one by one.", + "Start stacking {A} from the bottom to the top.", + "Stack the three {A} one above the other.", + "Position one {A} and stack two others on it.", + "Grab {A}, stack them one by one.", + "Stack the first {A}, then the second, then the third.", + "Grab the first {A} and stack the others on top.", + "Pick up the first {A} and layer the rest above it." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/stack_bowls_two.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/stack_bowls_two.json new file mode 100644 index 00000000..82d3b13b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/stack_bowls_two.json @@ -0,0 +1,69 @@ +{ + "full_description": "stack the two bowls on top of each other", + "schema": "{A} notifies the first bowl, {B} notifies the second bowl", + "preference": "num of words should not exceed 15. Degree of detail avg 5", + "seen": [ + "Lift {A}, put it on the table, and stack {B} above.", + "Pick up {A}, place it down, then position {B} on top.", + "Grab {A}, set it down, then place {B} over it.", + "Place {A} on the surface, then stack {B} above it.", + "Pick {A}, position on the table, and place {B} over it.", + "Lift {A}, lay it down, and stack {B} on top.", + "Take {A}, position it, then stack {B} directly above.", + "Grab {A}, then carefully place {B} on top.", + "Set {A} down, then add {B} over it.", + "Lift {B}, stack it precisely on top of {A}.", + "Pick {A}, then stack {B} on it.", + "Grab {A}, position it, and place {B} on top.", + "Align {A} and {B}, then stack {B} on {A}.", + "Place {A} down, then set {B} on top of {A}.", + "Position {A}, lift {B}, and place {B} on {A}.", + "Pick up {A}, grab {B}, and stack {B} on {A}.", + "Arrange {A}, lift {B} with the arm, and stack them.", + "Use the arm to place {A}, then stack {B} on top.", + "Position {A} with the arm, then set {B} above it.", + "Grasp {A}, then use the arm to align and stack {B}.", + "Grab {A}, then stack {B} on top.", + "Lift {A}, position {B} above it, and stack.", + "Set {B} over {A} to create the stack.", + "Position {B} carefully on {A}.", + "Hold {A} and stack {B} neatly atop.", + "Take {A}, align {B}, and stack them.", + "Place {A} down, then set {B} above it.", + "Lift {B} and position it neatly over {A}.", + "Stack {B} securely on top of {A}.", + "Align {B} over {A}, then complete the stack.", + "Place {A} down and stack {B} on top.", + "Lift {A}, grab {B}, and set {B} on {A}.", + "{A} goes at the bottom, {B} rests above.", + "Pick {A}, position {B}, and place it over {A}.", + "Put {A} first, then bring {B} to stack above.", + "Stack {B} carefully over {A}.", + "Set down {A} and drop {B} on top.", + "Grab {A}, lift it up, position {B}, and stack them.", + "Hold {A} steady and slide {B} on.", + "Take {A}, place {B} above, and let go.", + "Pick up {A} and set {B} on it.", + "Take {A}, place it, and stack {B} on top.", + "Lift {B}, then stack it on {A}.", + "Position {A} and stack {B} securely onto it.", + "Grab {A} with {a}, place it, and stack {B} with {b}.", + "Pick up {A} using {a}, then stack {B} using {b}.", + "With {a}, lift {A} and stack {B} using {b}.", + "Use {a} for {A}, place it, and stack {B} with {b}.", + "Grab {A} with {a}, put it down, then stack {B} onto it.", + "Lift {A} using {a}, set it down, and stack {B} using {b}." + ], + "unseen": [ + "Grab {A}, then place it on the surface.", + "Take {A}, set it down, and stack {B} on top.", + "Stack {B} on top of {A}.", + "Place {B} carefully over {A} to stack them.", + "Stack {B} directly over {A}.", + "Place {B} onto {A} to form a stack.", + "Grab {A}, then set {B} on top.", + "Pick up {A} and stack {B} above.", + "Grab {A} and stack {B} on it.", + "Place {A} down, then put {B} on top." + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/stamp_seal.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/stamp_seal.json new file mode 100644 index 00000000..13bb44a8 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/stamp_seal.json @@ -0,0 +1,69 @@ +{ + "full_description": "Grab the stamp and stamp onto the specific color mat", + "schema": "{A} notifies the stamp, {B} notifies the mat color, {a} notifies the arm to pick the stamp", + "preference": "num of words should not exceed 7.Degree of detail avg 5", + "seen": [ + "Use {a} to grab {A}", + "Press {A} firmly onto {B}", + "Position {A} over {B} and stamp", + "Grab {A}, align it with {B}", + "Use {a} to pick {A} for {B}", + "Lift {A} and press onto {B}", + "Grab {A} with {a} then stamp", + "Position {A} on {B}, apply pressure", + "Use {a} to grab {A}, press {B}", + "Pick {A} and press down on {B}", + "Stamp {B} after grabbing {A}.", + "With {a}, take {A} and mark {B}.", + "Place {A} onto {B} after grabbing.", + "Use {a} to lift {A}, then stamp {B}.", + "Pick {A} and apply it onto {B}.", + "Grab {A} with {a}, press it onto {B}.", + "Take {A} and stamp it on {B}.", + "With {a}, secure {A} and mark {B}.", + "Grab {A} and press on {B}.", + "Use {a}, take {A}, and apply to {B}.", + "Place {A} onto {B}", + "Grab {A} with {a} now", + "Stamp {A} on {B}", + "Use {a} to place {A}", + "Press {A} onto {B}", + "Grab {A} using {a}", + "Set {A} on {B}", + "With {a}, stamp {A}", + "Use {A} to stamp {B}", + "Grab {A} and press {B}", + "Use {a} to grab {A} and stamp", + "Stamp {B} after grabbing {A}", + "Pick up {A} and press onto {B}", + "Hold {A} with {a} and stamp {B}", + "Grab {A} to press onto {B}", + "Use {a} for {A} and stamp {B}", + "Pick {A} and stamp it on {B}", + "Press {A} onto {B} with {a}", + "Bring {A} to {B} and stamp", + "Use {a} to press {A} on {B}", + "{a} grabs {A}, stamps {B}", + "Pick {A} and press on {B}", + "{a} picks {A}, stamps {B}", + "Take {A} and stamp {B}", + "Use {A} to mark {B}", + "{a} holds {A}, presses {B}", + "Grab {A} and press {B}", + "Pick up {A}, stamp {B}", + "{a} uses {A} to stamp {B}", + "Hold {A} and press {B}" + ], + "unseen": [ + "Grab {A}, press onto {B}", + "Pick {A} and stamp on {B}", + "Pick {A} and press on {B}.", + "Use {a} to grab {A}, stamp {B}.", + "Pick {A} and stamp {B}", + "Use {a} to grab {A}", + "Grab {A} using {a} and stamp {B}", + "Pick {A} to stamp {B}", + "Grab {A} and stamp {B}", + "Stamp {B} using {A}" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/turn_switch.json b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/turn_switch.json new file mode 100644 index 00000000..c25d5686 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/task_instruction/turn_switch.json @@ -0,0 +1,69 @@ +{ + "full_description": "use the robotic arm to click the switch", + "schema": "{A} notifies the switch, {a} notifies the arm to click the switch", + "preference": "num of words should not exceed 7.Degree of detail avg 5", + "seen": [ + "Press {A} using {a}.", + "Activate {A} with {a}.", + "Move to {A} and click it.", + "Click {A} directly.", + "Press {A} to activate.", + "Engage {A} with a press.", + "Interact with {A}.", + "Push {A} via {a}.", + "Toggle {A} with {a}.", + "Engage {A} using {a}.", + "Engage {a} to press {A}", + "Press {A} directly", + "Use {a} to interact with {A}", + "Trigger {A} without referencing {a}", + "Command {a} to engage {A}", + "Set the switch at {A}", + "Activate {A} with robotic precision", + "Direct {a} to operate {A}", + "Switch {A} to its active state", + "Deploy {a} to press {A}", + "Engage {A} using the robotic arm", + "Press {A} using robotic control", + "Click {A} with robotic precision", + "Activate {A} directly", + "Press {A} via automation", + "Click {A} using automated tools", + "Deploy {a} to operate {A}", + "Use {a} to engage {A}", + "Set {a} to press {A}", + "Operate {A} with {a}", + "Activate {A} via {a}", + "Trigger {A} with precision", + "Tap {A} to engage it", + "Locate and press {A}", + "Initiate {A} by clicking", + "Engage {A} using {a}", + "Approach and tap {A} via {a}", + "Click on {A} precisely", + "Interact with {A} directly", + "Press {A} carefully with {a}", + "Directly interact with {A}", + "Activate the {A} using {a}", + "Press {A} with {a}", + "Click {A} to activate it", + "Operate {a} to engage {A}", + "Engage with {A} manually", + "Trigger {A} using {a}", + "Locate and press {A}", + "Use {a} for switching {A}", + "Activate {A} by pressing it" + ], + "unseen": [ + "Click {A} with {a}.", + "Use {a} to press {A}.", + "Click {A} using {a}", + "Activate the switch at {A}", + "Direct {a} to click {A}", + "Click {A} with {a}", + "Click {A} with {a}", + "Press {A} using {a}", + "Move and click the {A}", + "Use {a} to press {A}" + ] +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/utils/agent.py b/operator-images/builtin/holobrain-vendor/RoboTwin/description/utils/agent.py new file mode 100644 index 00000000..d86a49f4 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/utils/agent.py @@ -0,0 +1,48 @@ +from typing import List, Type, Optional +from pydantic import BaseModel, Field +import json +import os +from azure.ai.inference import ChatCompletionsClient +from azure.ai.inference.models import SystemMessage, UserMessage +from azure.core.credentials import AzureKeyCredential + +endpoint = "https://d-robotics.openai.azure.com/openai/deployments/gpt-4o" +model_name = "gpt-4o" + +# Get API key from environment variable +api_key = os.environ.get("AZURE_API_KEY") +if not api_key: + raise ValueError("AZURE_API_KEY environment variable is required but not set") + +client = ChatCompletionsClient( + endpoint=endpoint, + credential=AzureKeyCredential(api_key), +) + + +def generate(messages: List[dict], custom_format: Type[BaseModel]) -> Optional[BaseModel]: + strformat = custom_format.schema_json() + messages.append({ + "role": "system", + "content": "you shall output a json object with the following format: " + strformat, + }) + response = client.complete( + messages=messages, + max_tokens=4096, + temperature=0.8, + top_p=1.0, + model=model_name, + response_format="json_object", + ) + + json_content = response.choices[0].message.content + if json_content: + parsed_json = json.loads(json_content) + return (custom_format.parse_obj(parsed_json) + if hasattr(custom_format, "parse_obj") else custom_format.model_validate(parsed_json)) + + return None + + +if __name__ == "__main__": + pass diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/utils/clear_task_seen_unseen.py b/operator-images/builtin/holobrain-vendor/RoboTwin/description/utils/clear_task_seen_unseen.py new file mode 100644 index 00000000..54f59a0a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/utils/clear_task_seen_unseen.py @@ -0,0 +1,20 @@ +from argparse import ArgumentParser +import json + + +def clear_seen_unseen(task_name): + with open(f"./task_instruction/{task_name}.json", "r") as f: + task_info_json = f.read() + # print(task_info_json) + task_info = json.loads(task_info_json) + task_info["seen"] = [] + task_info["unseen"] = [] + with open(f"./task_instruction/{task_name}.json", "w") as f: + json.dump(task_info, f, indent=2, ensure_ascii=False) + + +if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument("task_name", type=str, default="beat_block_hammer") + args = parser.parse_args() + clear_seen_unseen(args.task_name) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/utils/convert_obj_glb.py b/operator-images/builtin/holobrain-vendor/RoboTwin/description/utils/convert_obj_glb.py new file mode 100644 index 00000000..7bfec6d5 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/utils/convert_obj_glb.py @@ -0,0 +1,101 @@ +import trimesh +import os +import numpy as np +import argparse +import traceback + + +def convert_obj_glb(source_dir): + """ + Convert all OBJ files in the given source directory to a single GLB file. + + Args: + source_dir: Directory containing OBJ files + output_visual_path: Path to the output directory + output_file: Output GLB file name (default: base0.glb) + + Returns: + bool: True if successful, False if an error occurs + """ + try: + texture_dir = os.path.join(source_dir, "textured_objs") + visual_dir = os.path.join(source_dir, "visual") + output_path = os.path.join(visual_dir, "base0.glb") + if os.path.exists(output_path): + print(f"File {output_path} already exists") + return True + if not os.path.exists(visual_dir): + os.makedirs(visual_dir) + # Create a scene to hold all meshes + scene = trimesh.Scene() + + # Find all .obj files in the directory + obj_files = [f for f in os.listdir(texture_dir) if f.endswith(".obj")] + + # Load each OBJ file and add it to the scene + for obj_file in obj_files: + file_path = os.path.join(texture_dir, obj_file) + try: + with open(file_path, "rb") as file_obj: + mesh = trimesh.load(file_obj, file_type="obj") + scene.add_geometry(mesh) + # print(f"Added mesh from {file_path}") + except Exception as e: + print(f"Error loading {file_path}: {e}") + return False + + # Export the scene as GLB + print(f"Exporting scene to {output_path}...") + scene.export(output_path) + print(f"Model successfully exported to {output_path}") + return True + except Exception as e: + print(f"An error occurred in convert_to_glb: {e}" + traceback.format_exc()) + return False + + +def is_digital(name): + """Check if a string contains only digits.""" + return name.isdigit() + + +def has_only_digital_subdirs(directory): + """Check if a directory contains only subdirectories with digital names.""" + if not os.path.isdir(directory): + return False + + subdirs = [item for item in os.listdir(directory) if os.path.isdir(os.path.join(directory, item))] + + # Return True if there are subdirs and all of them are digital + return len(subdirs) > 0 and all(is_digital(subdir) for subdir in subdirs) + + +if __name__ == "__main__": + # Set up argument parser + parser = argparse.ArgumentParser(description="Convert OBJ files to GLB.") + parser.add_argument( + "--object_dir", + type=str, + help="Directory containing single object (e.g., assets/objects/060_kitchenpot)", + ) + parser.add_argument( + "--scan_all", + action="store_true", + help="Scan all objects in assets/objects directory", + ) + args = parser.parse_args() + + total_conversions = 0 + + assets_path = "../assets/objects" + # Process each object directory in assets/objects + for obj_dir in os.listdir(assets_path): + obj_path = os.path.join(assets_path, obj_dir) + + # Check if it's a directory and has only digital subdirectories + if os.path.isdir(obj_path) and has_only_digital_subdirs(obj_path): + print(obj_path) + # for final_path in os.listdir(obj_path): + # convert_obj_glb(os.path.join(obj_path, final_path)) + + print(f"\nTotal completed GLB conversions: {total_conversions}") diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/utils/generate_episode_instructions.py b/operator-images/builtin/holobrain-vendor/RoboTwin/description/utils/generate_episode_instructions.py new file mode 100644 index 00000000..f3f00c9b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/utils/generate_episode_instructions.py @@ -0,0 +1,276 @@ +import json +import re +from typing import List, Dict, Any +import os +import argparse +import random +import yaml + +current_file_path = os.path.abspath(__file__) +parent_directory = os.path.dirname(current_file_path) + + +def extract_placeholders(instruction: str) -> List[str]: + """Extract all placeholders of the form {X} from an instruction.""" + placeholders = re.findall(r"{([^}]+)}", instruction) + return placeholders + +def filter_instructions(instructions: List[str], episode_params: Dict[str, str]) -> List[str]: + """ + Filter instructions to only include those that have all placeholders + matching the available episode parameters. No more, no less. + Also accept instructions that don't contain arm placeholder {[a-z]}. + """ + filtered_instructions = [] + random.shuffle(instructions) + + for instruction in instructions: + placeholders = extract_placeholders(instruction) + # Remove {} from episode_params keys for comparison + stripped_episode_params = {key.strip("{}"): value for key, value in episode_params.items()} + + # Get all arm-related parameters (single lowercase letters) + arm_params = {key for key in stripped_episode_params.keys() if len(key) == 1 and "a" <= key <= "z"} + non_arm_params = set(stripped_episode_params.keys()) - arm_params + + # Accept if we have exact match OR if the only missing parameters are arm parameters + if set(placeholders) == set(stripped_episode_params.keys()) or ( + # Special case: accept if the only difference is missing arm parameters + arm_params and set(placeholders).union(arm_params) == set(stripped_episode_params.keys()) and + not arm_params.intersection(set(placeholders))): + filtered_instructions.append(instruction) + + return filtered_instructions + + +def replace_placeholders(instruction: str, episode_params: Dict[str, str]) -> str: + """Replace all {X} placeholders in the instruction with corresponding values from episode_params. + For arm placeholders {[a-z]}, add 'the ' in front and ' arm' after the value. + If the value is a path to an existing JSON file, randomly choose one 'description' item and prepend 'the'. + If the value contains '\' or '/' but the file does not exist, print a bold warning. + """ + # Remove {} from episode_params keys for replacement + stripped_episode_params = {key.strip("{}"): value for key, value in episode_params.items()} + + for key, value in stripped_episode_params.items(): + placeholder = "{" + key + "}" + # Check if the value contains '\' or '/' + if "\\" in value or "/" in value: + json_path = os.path.join( + os.path.join(parent_directory, "../objects_description"), + value + ".json", + ) + if not os.path.exists(json_path): + print(f"\033[1mERROR: '{json_path}' looks like a description file, but does not exist.\033[0m") + exit() + + # Check if the value is a path to an existing JSON file + json_path = os.path.join(os.path.join(parent_directory, "../objects_description"), value + ".json") + if os.path.exists(json_path): + with open(json_path, "r") as f: + json_data = json.load(f) + # Randomly choose one description and prepend 'the' + description = random.choice(json_data.get("seen", [])) + value = f"the {description}" + # Check if the key is a single lowercase letter (arm placeholder) + elif len(key) == 1 and "a" <= key <= "z": + value = f"the {value} arm" + else: + value = f"{value}" + + instruction = instruction.replace(placeholder, value) + + return instruction + + +def replace_placeholders_unseen(instruction: str, episode_params: Dict[str, str]) -> str: + """Similar to replace_placeholders but uses 'unseen' descriptions from JSON files. + For arm placeholders {[a-z]}, add 'the ' in front and ' arm' after the value. + If the value is a path to an existing JSON file, randomly choose one 'unseen' description and prepend 'the'. + If the value contains '\' or '/' but the file does not exist, print a bold warning. + """ + # Remove {} from episode_params keys for replacement + stripped_episode_params = {key.strip("{}"): value for key, value in episode_params.items()} + + for key, value in stripped_episode_params.items(): + placeholder = "{" + key + "}" + # Check if the value contains '\' or '/' + if "\\" in value or "/" in value: + json_path = os.path.join( + os.path.join(parent_directory, "../objects_description"), + value + ".json", + ) + if not os.path.exists(json_path): + print(f"\033[1mERROR: '{json_path}' looks like a description file, but does not exist.\033[0m") + exit() + + # Check if the value is a path to an existing JSON file + json_path = os.path.join(os.path.join(parent_directory, "../objects_description"), value + ".json") + if os.path.exists(json_path): + with open(json_path, "r") as f: + json_data = json.load(f) + # Randomly choose one unseen description and prepend 'the' + if "unseen" in json_data and json_data["unseen"]: + description = random.choice(json_data.get("unseen", [])) + value = f"the {description}" + else: + # Fall back to seen descriptions if unseen is empty + description = random.choice(json_data.get("seen", [])) + value = f"the {description}" + # Check if the key is a single lowercase letter (arm placeholder) + elif len(key) == 1 and "a" <= key <= "z": + value = f"the {value} arm" + else: + value = f"{value}" + + instruction = instruction.replace(placeholder, value) + + return instruction + + +def load_task_instructions(task_name: str) -> Dict[str, Any]: + """Load the task instructions from the JSON file.""" + file_path = os.path.join(parent_directory, f"../task_instruction/{task_name}.json") + with open(file_path, "r") as f: + task_data = json.load(f) + return task_data + + +def load_scene_info(task_name: str, setting: str, scene_info_path: str) -> Dict[str, Dict]: + """Load the scene info from the JSON file in the data directory.""" + file_path = os.path.join(parent_directory, f"../../{scene_info_path}/{task_name}/{setting}/scene_info.json") + try: + with open(file_path, "r") as f: + scene_data = json.load(f) + return scene_data + except FileNotFoundError: + print(f"\033[1mERROR: Scene info file '{file_path}' not found.\033[0m") + exit(1) + except json.JSONDecodeError: + print(f"\033[1mERROR: Scene info file '{file_path}' contains invalid JSON.\033[0m") + exit(1) + + +def extract_episodes_from_scene_info(scene_info: Dict) -> List[Dict[str, str]]: + """Extract episode parameters from scene_info.""" + episodes = [] + for episode_key, episode_data in scene_info.items(): + if "info" in episode_data: + episodes.append(episode_data["info"]) + else: + episodes.append(dict()) + return episodes + + +def save_episode_descriptions(task_name: str, setting: str, generated_descriptions: List[Dict]): + """Save generated descriptions to output files.""" + output_dir = os.path.join(parent_directory, f"../../data/{task_name}/{setting}/instructions") + os.makedirs(output_dir, exist_ok=True) + + for episode_desc in generated_descriptions: + episode_index = episode_desc["episode_index"] + output_file = os.path.join(output_dir, f"episode{episode_index}.json") + + with open(output_file, "w") as f: + json.dump( + { + "seen": episode_desc.get("seen", []), + "unseen": episode_desc.get("unseen", []), + }, + f, + indent=2, + ) + +def generate_episode_descriptions(task_name: str, episodes: List[Dict[str, str]], max_descriptions: int = 1000000): + """ + Generate descriptions for episodes by replacing placeholders in instructions with parameter values. + For each episode, filter instructions that have matching placeholders and generate up to + max_descriptions by replacing placeholders with parameter values. + Now also generates unseen descriptions. + """ + # Load task instructions + task_data = load_task_instructions(task_name) + seen_instructions = task_data.get("seen", []) + unseen_instructions = task_data.get("unseen", []) + + # Store generated descriptions for each episode + all_generated_descriptions = [] + + # Process each episode + for i, episode in enumerate(episodes): + # Filter instructions that have all placeholders matching episode parameters + filtered_seen_instructions = filter_instructions(seen_instructions, episode) + filtered_unseen_instructions = filter_instructions(unseen_instructions, episode) + + if filtered_seen_instructions == [] and filtered_unseen_instructions == []: + print(f"Episode {i}: No valid instructions found") + continue + + # Generate seen descriptions by replacing placeholders + seen_episode_descriptions = [] + flag_seen = True + while (len(seen_episode_descriptions) < max_descriptions and flag_seen and filtered_seen_instructions): + + for instruction in filtered_seen_instructions: + if len(seen_episode_descriptions) >= max_descriptions: + flag_seen = False + break + description = replace_placeholders(instruction, episode) + seen_episode_descriptions.append(description) + + # Generate unseen descriptions by replacing placeholders + unseen_episode_descriptions = [] + flag_unseen = True + while (len(unseen_episode_descriptions) < max_descriptions and flag_unseen and filtered_unseen_instructions): + for instruction in filtered_unseen_instructions: + if len(unseen_episode_descriptions) >= max_descriptions: + flag_unseen = False + break + description = replace_placeholders_unseen(instruction, episode) + unseen_episode_descriptions.append(description) + + all_generated_descriptions.append({ + "episode_index": i, + "seen": seen_episode_descriptions, + "unseen": unseen_episode_descriptions, + }) + + return all_generated_descriptions + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Generate episode descriptions by replacing placeholders") + parser.add_argument( + "task_name", + type=str, + help="Name of the task (JSON file name without extension)", + ) + parser.add_argument( + "setting", + type=str, + help="Setting name used to construct the data directory path", + ) + parser.add_argument( + "max_num", + type=int, + default=100, + help="Maximum number of descriptions per episode", + ) + + args = parser.parse_args() + setting_file = os.path.join( + parent_directory, f"../../task_config/{args.setting}.yml" + ) + with open(setting_file, "r", encoding="utf-8") as f: + args_dict = yaml.load(f.read(), Loader=yaml.FullLoader) + + # Load scene info and extract episode parameters + scene_info = load_scene_info(args.task_name, args.setting, args_dict['save_path']) + episodes = extract_episodes_from_scene_info(scene_info) + + # Generate descriptions + results = generate_episode_descriptions(args.task_name, episodes, args.max_num) + + # Save results to output files + save_episode_descriptions(args.task_name, args.setting, results) + print("Successfully Saved Instructions") \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/utils/generate_object_description.py b/operator-images/builtin/holobrain-vendor/RoboTwin/description/utils/generate_object_description.py new file mode 100644 index 00000000..b36fce1a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/utils/generate_object_description.py @@ -0,0 +1,192 @@ +import json +from agent import * +from argparse import ArgumentParser +from get_image_from_glb import * +import os +import base64 +import pprint +import time +import random + + +class subPart(BaseModel): + name: str + color: str + shape: str + size: str + material: str + functionality: str + texture: str + + +class ObjDescFormat(BaseModel): + raw_description: str = Field(description="the name of the object,without index and '_'") + wholePart: subPart = Field(description="the object as a whole") + subParts: List[subPart] = Field( + description="the deformable subparts of the object.If the object is not deformable, leave empty here") + description: List[str] = Field(description="several different text descriptions describing this same object here") + # val_description:List[str]=Field(description="similar to descriptions, used for validation") + + +with open("./_generate_object_prompt.txt", "r") as f: + system_prompt = f.read() + + +def save_json(save_dir, glb_file_name, ObjDescResult): + os.makedirs(save_dir, exist_ok=True) + # Remove .glb extension from the filename + base_name = glb_file_name.replace(".glb", "") + save_path = f"{save_dir}/{base_name}.json" + + # Get all descriptions + all_descriptions = ObjDescResult.description.copy() + all_descriptions.sort(key=len) + # Randomly select 5 indices for validation set + val_indices = random.sample(range(len(all_descriptions)), 3) + + # Separate validation and training descriptions based on indices + shuffle_val = [all_descriptions[i] for i in val_indices] + shuffle_train = [all_descriptions[i] for i in range(len(all_descriptions)) if i not in val_indices] + + # Sort both validation and training descriptions by character length + shuffle_val.sort(key=len) + shuffle_train.sort(key=len) + + # 将字典保存为 JSON 文件 + desc_dict = { + "raw_description": ObjDescResult.raw_description, + "seen": shuffle_train, + "unseen": shuffle_val, + } + with open(save_path, "w", encoding="utf-8") as file: + json.dump(desc_dict, file, ensure_ascii=False, indent=4) + print(json.dumps(desc_dict, indent=2, ensure_ascii=False)) + + +def save_image(save_dir, glb_file_name, imgstr): + os.makedirs(save_dir, exist_ok=True) + save_image_path = f"{save_dir}/{glb_file_name}.png" + with open(save_image_path, "wb") as f: + # Convert the Base64 string to bytes before writing + img_data = base64.b64decode(imgstr) + f.write(img_data) + + +def make_prompt_generate(imgStr, object_name): + messages = [ + { + "role": "system", + "content": system_prompt + }, + { + "role": + "user", + "content": [ + { + "type": "text", + "text": f"THE OBJECT IS A {object_name}" + }, + { + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{imgStr}" + }, + }, + ], + }, + ] + result = generate(messages, ObjDescFormat) + result_dict = result.model_dump() + print( + json.dumps( + { + "wholePart": result_dict["wholePart"], + "subParts": result_dict["subParts"], + }, + indent=2, + ensure_ascii=False, + )) + return result + + +def generate_obj_description(object_name, glb_file_name): + time_start = time.time() + object_file_path = f"../assets/objects/{object_name}/visual/{glb_file_name}" + save_dir = f"./objects_description/{object_name}" + result_img_path = f"{save_dir}/{glb_file_name}.png" + if not os.path.exists(result_img_path): + imgstr = get_image_from_glb(object_file_path) + print(f"{object_name} {glb_file_name} saving image", time.time() - time_start) + time_start = time.time() + save_image(save_dir, glb_file_name, imgstr) + else: + print( + f'{object_name} {glb_file_name} using existing image: {result_img_path}. If errors like "Message: Invalid image data." occurs, please delete the image and rerun the script' + ) + with open(result_img_path, "rb") as f: + imgstr = base64.b64encode(f.read()).decode("utf-8") + print(f"{object_name} {glb_file_name} start generating", time.time() - time_start) + time_start = time.time() + result = make_prompt_generate(imgstr, object_name) + print( + f"{object_name} {glb_file_name} generated {len(str(result.model_dump()))} descriptions ", + time.time() - time_start, + ) + save_json(save_dir, glb_file_name, result) + + +if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument("object_name", type=str, nargs="?", default=None, help="Object name to process") + parser.add_argument("--index", type=int, default=None, help="Specific object index to process") + parser.add_argument("--store_png", action="store_true", help="Store PNG files after generation") + usr_args = parser.parse_args() + + object_name = usr_args.object_name + object_index = usr_args.index + clear_png = not usr_args.store_png + + if object_name is None: # process all objects + objects_dir = "../assets/objects" + results_dir = "./objects_description" + for object_name in sorted(os.listdir(objects_dir)): + parts = object_name.split("_") + if not (len(parts) == 2): + continue + object_dir = os.path.join(objects_dir, object_name) + if os.path.isdir(object_dir): + visual_dir = os.path.join(object_dir, "visual") + if os.path.exists(visual_dir): + print(f"Processing object: {object_name}") + glb_files = [file for file in os.listdir(visual_dir) if file.endswith(".glb")] + for glb_file in sorted(glb_files): + if os.path.exists(os.path.join( + results_dir, + object_name, + glb_file.replace(".glb", ".json"), + )): + continue + generate_obj_description(object_name, glb_file) + if clear_png: + png_path = (f"./objects_description/{object_name}/{glb_file}.png") + if os.path.exists(png_path): + os.remove(png_path) + print(f"Deleted: {png_path}") + elif object_index is None: # all type for specific object + folder_path = f"../assets/objects/{object_name}/visual" + files_and_folders = os.listdir(folder_path) + glb_files = [file for file in files_and_folders if file.endswith(".glb")] + for glb_file in glb_files: + generate_obj_description(object_name, glb_file) + if clear_png: + png_path = f"./objects_description/{object_name}/{glb_file}.png" + if os.path.exists(png_path): + os.remove(png_path) + print(f"Deleted: {png_path}") + else: # specific object and index + generate_obj_description(object_name, f"base{object_index}.glb") + if clear_png: + png_path = f"./objects_description/{object_name}/base{object_index}.glb.png" + if os.path.exists(png_path): + os.remove(png_path) + print(f"Deleted: {png_path}") diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/utils/generate_task_description.py b/operator-images/builtin/holobrain-vendor/RoboTwin/description/utils/generate_task_description.py new file mode 100644 index 00000000..c7e85e15 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/utils/generate_task_description.py @@ -0,0 +1,112 @@ +import json +from agent import * +from argparse import ArgumentParser +import os + +with open("./_generate_task_prompt.txt", "r") as f: + system_prompt = f.read() + + +class Instruction(BaseModel): + content: str = Field(description="the instruction for the task") + degreeOfDetail: int = Field(description="the degree of detail for the instruction, from 1 to 10") + armMention: bool = Field(description="whether the instruction mentions arm, whether by schema or by fixed text") + numOfWords: int = Field(description="the number of words in the instruction") + + +class InstructionFormat(BaseModel): + stepsOfTask: List[str] = Field( + description= + "split the task into small steps, and make sure each step is explicitly or implicitly mentioned in each of the instructions.Avoid using adjectives in it!" + ) + instructions: List[Instruction] = Field( + description="several different text instructions describing this same task here") + + +def make_prompt_generate(detailed_task, preferences, schema, instruction_num): + system_prompt_schema = "" + if schema: + with open("./_generate_task_prompt_schema.txt", "r") as f: + system_prompt_schema = f.read() + messages = [ + { + "role": "system", + "content": system_prompt + "\n" + system_prompt_schema + }, + { + "role": + "user", + "content": [ + # {"type":"image_url","image_url":{"url":f"data:image/png;base64,{imgStr}"}, + { + "type": "text", + "text": f"The detailed task description for you to abstract is {detailed_task}", + }, + { + "type": "text", + "text": f"For each instruction, you should follow the preference: {preferences}", + }, + { + "type": "text", + "text": f"Generate {instruction_num} alternative descriptions based on the input.", + }, + ], + }, + ] + if schema: + messages[1]["content"].append({ + "type": "text", + "text": f"The object schema for you to abstract is {schema}", + }) + result = generate(messages, InstructionFormat) + result_dict = result.model_dump() + print(json.dumps(result_dict, indent=2, ensure_ascii=False)) + insList = [] + for ins in result.instructions: + insList.append(ins.content) + return insList + + +def generate_task_description(task_name, instruction_num): + with open(f"./task_instruction/{task_name}.json", "r") as f: + task_info_json = f.read() + # print(task_info_json) + task_info = json.loads(task_info_json) + if "seen" not in task_info.keys(): + task_info["seen"] = [] + if "unseen" not in task_info.keys(): + task_info["unseen"] = [] + for required_keys in [ + "full_description", + "preference", + ]: # schema can be empty to disable it + if (not task_info.get(required_keys, "") or task_info.get(required_keys, "") == ""): + print(f"{required_keys} is not in the ./task_instruction/{task_name}.json or is empty") + return + result = make_prompt_generate( + task_info["full_description"], + task_info["preference"], + task_info["schema"], + instruction_num, + ) + print(f'{task_name} generated {len(result)} descriptions with length {len("".join(result))}') + task_info["seen"].extend(result[2:]) + task_info["unseen"].extend(result[0:2]) + # task_info['seen'] = result[2:] + # task_info['unseen'] = result[0:2] + with open(f"./task_instruction/{task_name}.json", "w") as f: + json.dump(task_info, f, indent=2, ensure_ascii=False) + + +if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument("task_name", type=str, default="beat_block_hammer") + parser.add_argument("instruction_num", type=int, default=11) + usr_args = parser.parse_args() + task_name = usr_args.task_name + instruction_num = usr_args.instruction_num + if instruction_num % 12 != 0: + print("instruction_num should be divisible by 12") + exit() + for i in range(instruction_num // 12): + generate_task_description(task_name, 12) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/description/utils/get_image_from_glb.py b/operator-images/builtin/holobrain-vendor/RoboTwin/description/utils/get_image_from_glb.py new file mode 100644 index 00000000..f5f885f0 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/description/utils/get_image_from_glb.py @@ -0,0 +1,898 @@ +import argparse +import os +import sys +import trimesh +import numpy as np +import PIL.Image +from io import BytesIO +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from mpl_toolkits.mplot3d import Axes3D +import base64 +import random +from typing import List, Tuple, Optional, Union +import traceback + +os.environ["PYGLET_HEADLESS"] = "1" +os.environ["PYOPENGL_PLATFORM"] = "egl" +PI = np.pi + + +class ModelLoader: + """Class responsible for loading 3D models from files.""" + + @staticmethod + def load_from_glb(file_path: str) -> trimesh.Scene: + """ + Load a 3D model from a GLB file. + Args: + file_path: Path to the .glb file + Returns: + trimesh.Scene object containing the model + Raises: + FileNotFoundError: If the file doesn't exist + ValueError: If the file can't be loaded as a GLB + """ + if not os.path.exists(file_path): + raise FileNotFoundError(f"Model file not found: {file_path}") + try: + with open(file_path, "rb") as file_obj: + mesh = trimesh.load(file_obj, file_type="glb") + return trimesh.Scene(mesh) + except Exception as e: + raise ValueError(f"Failed to load GLB file: {str(e)}") + + +class BoundingBox: + """Class for creating and manipulating bounding boxes around 3D models.""" + + def __init__(self, scene: trimesh.Scene, scale_factor: float = 1.0): + """ + Initialize BoundingBox with a scene. + Args: + scene: trimesh.Scene object + scale_factor: Factor to scale the bounding box by + """ + self.scene = scene + self.centroid = scene.centroid + self.bounds = scene.bounds + self.scale_factor = scale_factor + self.min_bound, self.max_bound = self._calculate_scaled_bounds() + + def _calculate_scaled_bounds(self) -> Tuple[np.ndarray, np.ndarray]: + """ + Calculate the scaled bounds of the bounding box. + Returns: + Tuple of (min_bound, max_bound) arrays + """ + min_bound, max_bound = self.bounds + original_half_size = (max_bound - min_bound) / 2.0 + scaled_half_size = original_half_size * self.scale_factor + scaled_min_bound = self.centroid - scaled_half_size + scaled_max_bound = self.centroid + scaled_half_size + return scaled_min_bound, scaled_max_bound + + def add_to_scene(self) -> trimesh.Scene: + """ + Add bounding box visualization to the scene. + Returns: + Updated scene with bounding box + """ + corners = np.array([ + [self.min_bound[0], self.min_bound[1], self.min_bound[2]], + [self.max_bound[0], self.min_bound[1], self.min_bound[2]], + [self.max_bound[0], self.max_bound[1], self.min_bound[2]], + [self.min_bound[0], self.max_bound[1], self.min_bound[2]], + [self.min_bound[0], self.min_bound[1], self.max_bound[2]], + [self.max_bound[0], self.min_bound[1], self.max_bound[2]], + [self.max_bound[0], self.max_bound[1], self.max_bound[2]], + [self.min_bound[0], self.max_bound[1], self.max_bound[2]], + ]) + edges = np.array([ + [0, 1], + [1, 2], + [2, 3], + [3, 0], + [4, 5], + [5, 6], + [6, 7], + [7, 4], + [0, 4], + [1, 5], + [2, 6], + [3, 7], + ]) + for edge in edges: + line_points = np.array([corners[edge[0]], corners[edge[1]]]) + line = trimesh.path.Path3D(entities=[trimesh.path.entities.Line([0, 1])], vertices=line_points) + self.scene.add_geometry(line, node_name=f"bound_edge_{edge[0]}_{edge[1]}") + return self.scene + + def calculate_face_centers(self) -> List[Tuple[float, float, float]]: + """ + Calculate the center points of each face of the bounding box. + Returns: + List of face center coordinates + """ + return [ + ( + self.min_bound[0], + (self.min_bound[1] + self.max_bound[1]) / 2, + (self.min_bound[2] + self.max_bound[2]) / 2, + ), + ( + self.max_bound[0], + (self.min_bound[1] + self.max_bound[1]) / 2, + (self.min_bound[2] + self.max_bound[2]) / 2, + ), + ( + (self.min_bound[0] + self.max_bound[0]) / 2, + self.min_bound[1], + (self.min_bound[2] + self.max_bound[2]) / 2, + ), + ( + (self.min_bound[0] + self.max_bound[0]) / 2, + self.max_bound[1], + (self.min_bound[2] + self.max_bound[2]) / 2, + ), + ( + (self.min_bound[0] + self.max_bound[0]) / 2, + (self.min_bound[1] + self.max_bound[1]) / 2, + self.min_bound[2], + ), + ( + (self.min_bound[0] + self.max_bound[0]) / 2, + (self.min_bound[1] + self.max_bound[1]) / 2, + self.max_bound[2], + ), + ] + + +class VisualElements: + """Class for creating visual elements like arrows and markers for scene visualization.""" + + def __init__(self, scene: trimesh.Scene, bounding_box: BoundingBox): + """ + Initialize VisualElements with a scene and bounding box. + Args: + scene: trimesh.Scene object + bounding_box: BoundingBox object + """ + self.scene = scene + self.bounding_box = bounding_box + self.face_colors = [ + [255, 0, 0, 255], + [0, 255, 0, 255], + [0, 0, 255, 255], + [255, 255, 0, 255], + [255, 0, 255, 255], + [0, 255, 255, 255], + ] + self.centroid_color = [255, 255, 255, 255] + + def create_arrow( + self, + start_point: Tuple[float, float, float], + end_point: Tuple[float, float, float], + color: List[int], + ) -> Optional[trimesh.Trimesh]: + """ + Create an arrow pointing from start_point to end_point. + Args: + start_point: Starting coordinates of the arrow + end_point: Ending coordinates of the arrow + color: RGBA color for the arrow + Returns: + Arrow mesh or None if creation fails + """ + direction = np.array(end_point) - np.array(start_point) + distance = np.linalg.norm(direction) + if distance <= 0: + return None + direction = direction / distance + box_size = np.linalg.norm(self.bounding_box.max_bound - self.bounding_box.min_bound) + arrow_shaft_radius = box_size * 0.005 + arrow_head_radius = arrow_shaft_radius * 3 + arrow_head_length = box_size * 0.03 + arrow_length = min(distance * 0.7, box_size * 0.3) + shaft_length = arrow_length - arrow_head_length + if shaft_length <= 0: + return None + shaft = trimesh.creation.cylinder(radius=arrow_shaft_radius, height=shaft_length, sections=12) + shaft.vertices[:, 2] -= shaft_length / 2 + head = trimesh.creation.cone(radius=arrow_head_radius, height=arrow_head_length, sections=12) + head_transform = np.eye(4) + head_transform[:3, 3] = [0, 0, shaft_length] + head.apply_transform(head_transform) + arrow = trimesh.util.concatenate([shaft, head]) + arrow.visual.face_colors = color + current_direction = np.array([0, 0, 1]) + rotation_axis = np.cross(current_direction, direction) + rotation_axis_norm = np.linalg.norm(rotation_axis) + transform = np.eye(4) + if rotation_axis_norm > 1e-6: + rotation_axis = rotation_axis / rotation_axis_norm + rotation_angle = np.arccos(np.clip(np.dot(current_direction, direction), -1.0, 1.0)) + rotation = trimesh.transformations.rotation_matrix(rotation_angle, rotation_axis) + transform[:3, :3] = rotation[:3, :3] + else: + if np.dot(current_direction, direction) < 0: + rotation = trimesh.transformations.rotation_matrix(np.pi, [1, 0, 0]) + transform[:3, :3] = rotation[:3, :3] + transform[:3, 3] = start_point + arrow.apply_transform(transform) + return arrow + + def add_face_arrows(self) -> trimesh.Scene: + """ + Add arrows pointing from each face center to the centroid. + Returns: + Updated scene with face arrows + """ + face_centers = self.bounding_box.calculate_face_centers() + centroid = self.bounding_box.centroid + for i, center in enumerate(face_centers): + arrow = self.create_arrow(center, centroid, self.face_colors[i % len(self.face_colors)]) + if arrow is not None: + self.scene.add_geometry(arrow, node_name=f"face_arrow_{i}") + return self.scene + + def add_centroid_marker(self) -> trimesh.Scene: + """ + Add a marker for the centroid. + Returns: + Updated scene with centroid marker + """ + box_size = np.linalg.norm(self.bounding_box.max_bound - self.bounding_box.min_bound) + radius = 0.015 * box_size + centroid_sphere = trimesh.primitives.Sphere(radius=radius, center=self.bounding_box.centroid) + centroid_sphere.visual.face_colors = self.centroid_color + self.scene.add_geometry(centroid_sphere, node_name="centroid") + return self.scene + + +class SceneRenderer: + """Class for rendering 3D scenes to images.""" + + def __init__(self, scene: trimesh.Scene): + """ + Initialize SceneRenderer with a scene. + Args: + scene: trimesh.Scene object to render + """ + self.scene = scene + + def render_image( + self, + resolution: Tuple[int, int] = (1024, 1024), + output_path: str = "object.png", + ) -> str: + """ + Render the scene and save the image. + Args: + resolution: Tuple of (width, height) for the output image + output_path: Path to save the rendered image + Returns: + Path to the saved image + """ + try: + png = self.scene.save_image(resolution=resolution, visible=True) + with open(output_path, "wb") as f: + f.write(png) + return output_path + except Exception as e: + print(f"Error rendering scene: {str(e)}") + raise + + def render_from_direction( + self, + camera_position: Tuple[float, float, float], + resolution: Tuple[int, int] = (1024, 1024), + output_path: str = "object.png", + ) -> str: + """ + Render the scene from a specific camera position. + Args: + camera_position: Position of the camera + resolution: Tuple of (width, height) for the output image + output_path: Path to save the rendered image + Returns: + Path to the saved image + """ + view_scene = self.scene.copy() + centroid = view_scene.centroid + camera_target = centroid + forward = np.array(camera_position) - np.array(camera_target) + distance = np.linalg.norm(forward) + if distance > 0: + forward = forward / distance + else: + forward = np.array([0, 0, 1]) + world_up = np.array([0, 0, 1]) + right = np.cross(world_up, forward) + if np.linalg.norm(right) > 0: + right = right / np.linalg.norm(right) + else: + right = np.array([1, 0, 0]) + camera_up = np.cross(forward, right) + rotation = np.eye(4) + rotation[:3, 0] = right + rotation[:3, 1] = camera_up + rotation[:3, 2] = forward + translation = np.eye(4) + translation[:3, 3] = camera_position + camera_transform = np.dot(translation, rotation) + view_scene.camera.fov = [60, 60] + view_scene.camera.resolution = resolution + view_scene.camera_transform = camera_transform + try: + png = view_scene.save_image(resolution=resolution, visible=True) + with open(output_path, "wb") as f: + f.write(png) + return output_path + except Exception as e: + print(f"Error rendering scene from direction: {str(e)}") + raise + + def render_from_position_and_direction( + self, + camera_position: Tuple[float, float, float], + camera_direction: Tuple[float, float, float], + resolution: Tuple[int, int] = (1024, 1024), + output_path: str = "object.png", + return_png: bool = False, + ) -> Union[str, bytes]: + """ + Render the scene from a specific camera position pointing in a specific direction. + Args: + camera_position: Position of the camera + camera_direction: Direction vector the camera is pointing (not normalized) + resolution: Tuple of (width, height) for the output image + output_path: Path to save the rendered image + return_png: If True, return the PNG data instead of saving to file + Returns: + Path to the saved image or PNG data as bytes if return_png=True + """ + view_scene = self.scene.copy() + forward = np.array(camera_direction) + distance = np.linalg.norm(forward) + if distance > 0: + forward = forward / distance + else: + forward = np.array([0, 0, 1]) + world_up = np.array([0, 0, 1]) + right = np.cross(world_up, forward) + if np.linalg.norm(right) > 0: + right = right / np.linalg.norm(right) + else: + right = np.array([1, 0, 0]) + camera_up = np.cross(forward, right) + rotation = np.eye(4) + rotation[:3, 0] = right + rotation[:3, 1] = camera_up + rotation[:3, 2] = forward + translation = np.eye(4) + translation[:3, 3] = camera_position + camera_transform = np.dot(translation, rotation) + view_scene.camera.fov = [60, 60] + view_scene.camera.resolution = resolution + view_scene.camera_transform = camera_transform + try: + png = view_scene.save_image(resolution=resolution, visible=True) + if return_png: + return png + else: + with open(output_path, "wb") as f: + f.write(png) + return output_path + except Exception as e: + print(f"Error rendering scene from position and direction: {str(e)}{traceback.format_exc()} ") + raise + + +class GLBRenderer: + """Class that combines all functionality to render images from GLB files.""" + + @staticmethod + def render_single_view( + file_path: str, + resolution: Tuple[int, int] = (1024, 1024), + show_bounds: bool = False, + show_arrows: bool = False, + output_path: str = "object.png", + ) -> str: + """ + Render a single view of a GLB model with visualization elements. + Args: + file_path: Path to the .glb file + resolution: Tuple of (width, height) for the output image + show_bounds: Whether to show bounding box + show_arrows: Whether to show arrows and centroid marker + output_path: Path to save the rendered image + Returns: + Path to the saved image + """ + try: + scene = ModelLoader.load_from_glb(file_path) + if show_bounds or show_arrows: + scale_factor = 1.0 if show_bounds else 8.0 + bbox = BoundingBox(scene, scale_factor) + if show_bounds: + scene = bbox.add_to_scene() + print(f"Raw bounding box bounds: [{bbox.min_bound}, {bbox.max_bound}]") + if show_arrows: + visuals = VisualElements(scene, bbox) + scene = visuals.add_face_arrows() + scene = visuals.add_centroid_marker() + renderer = SceneRenderer(scene) + image_path = renderer.render_image(resolution, output_path) + print(f"Image saved to {image_path}") + return image_path + except Exception as e: + print(f"Error rendering GLB file: {str(e)}") + raise + + @staticmethod + def render_six_views( + file_path: str, + resolution: Tuple[int, int] = (1024, 1024), + output_prefix: str = "object", + show_bounds: bool = False, + show_arrows: bool = False, + ) -> List[str]: + """ + Render six orthogonal views of a GLB model. + Args: + file_path: Path to the .glb file + resolution: Tuple of (width, height) for the output images + output_prefix: Prefix for output image filenames + show_bounds: Whether to show bounding box + show_arrows: Whether to show arrows and centroid marker + Returns: + List of paths to the saved images + """ + try: + scene = ModelLoader.load_from_glb(file_path) + scale_factor = 1.0 if show_bounds else 8.0 + bbox = BoundingBox(scene, scale_factor) + if show_bounds: + scene = bbox.add_to_scene() + print(f"Raw bounding box bounds: [{bbox.min_bound}, {bbox.max_bound}]") + if show_arrows: + visuals = VisualElements(scene, bbox) + scene = visuals.add_face_arrows() + scene = visuals.add_centroid_marker() + face_centers = bbox.calculate_face_centers() + direction_names = ["front", "back", "left", "right", "bottom", "top"] + image_paths = [] + renderer = SceneRenderer(scene) + for i, center in enumerate(face_centers): + image_path = f"{output_prefix}_{direction_names[i]}.png" + renderer.render_from_direction(center, resolution, image_path) + image_paths.append(image_path) + print(f"Image saved to {image_path}") + return image_paths + except Exception as e: + print(f"Error rendering six views: {str(e)}") + raise + + @staticmethod + def render_from_arrows( + file_path: str, + arrow_positions_and_directions: List[Tuple[Tuple[float, float, float], Tuple[float, float, float]]], + resolution: Tuple[int, int] = (1024, 1024), + output_prefix: str = "arrow_view", + ) -> List[str]: + """ + Render views from arbitrary camera positions and directions. + Args: + file_path: Path to the .glb file + arrow_positions_and_directions: List of (position, direction) tuples + resolution: Tuple of (width, height) for the output images + output_prefix: Prefix for output image filenames + Returns: + List of paths to the saved images + """ + try: + scene = ModelLoader.load_from_glb(file_path) + image_paths = [] + renderer = SceneRenderer(scene) + for i, (position, direction) in enumerate(arrow_positions_and_directions): + image_path = f"{output_prefix}_{i}.png" + renderer.render_from_position_and_direction(position, direction, resolution, image_path) + image_paths.append(image_path) + print(f"Image saved to {image_path}") + return image_paths + except Exception as e: + print(f"Error rendering from arrows: {str(e)}") + raise + + @staticmethod + def render_six_arrow_views( + file_path: str, + resolution: Tuple[int, int] = (1024, 1024), + output_prefix: str = "arrow_view", + show_bounds: bool = False, + show_arrows: bool = False, + ) -> List[str]: + """ + Render six views using calculated arrow positions and directions. + Args: + file_path: Path to the .glb file + resolution: Tuple of (width, height) for the output images + output_prefix: Prefix for output image filenames + show_bounds: Whether to show bounding box + show_arrows: Whether to show arrows and centroid marker + Returns: + List of paths to the saved images + """ + try: + scene = ModelLoader.load_from_glb(file_path) + scale_factor = 1.0 if show_bounds else 8.0 + bbox = BoundingBox(scene, scale_factor) + if show_bounds: + scene = bbox.add_to_scene() + print(f"Raw bounding box bounds: [{bbox.min_bound}, {bbox.max_bound}]") + if show_arrows: + visuals = VisualElements(scene, bbox) + scene = visuals.add_face_arrows() + scene = visuals.add_centroid_marker() + arrows = GLBRenderer.calculate_six_arrows(scene) + direction_names = ["front", "back", "left", "right", "bottom", "top"] + image_paths = [] + renderer = SceneRenderer(scene) + for i, (position, direction) in enumerate(arrows): + image_path = f"{output_prefix}_{direction_names[i]}.png" + renderer.render_from_position_and_direction(position, direction, resolution, image_path) + image_paths.append(image_path) + print(f"Image saved to {image_path}") + return image_paths + except Exception as e: + print(f"Error rendering six arrow views: {str(e)}") + raise + + @staticmethod + def calculate_six_arrows( + scene: trimesh.Scene, ) -> List[Tuple[Tuple[float, float, float], Tuple[float, float, float]]]: + """ + Calculate six camera positions and directions based on the scene's bounding box. + Args: + scene: The 3D scene + Returns: + List of (position, direction) tuples for camera placement + """ + bbox = BoundingBox(scene) + centroid = bbox.centroid + face_centers = bbox.calculate_face_centers() + arrows = [] + for center in face_centers: + position = center + direction = np.array(center) - np.array(centroid) + arrows.append((position, tuple(direction))) + return arrows + + @staticmethod + def render_from_polaris_position( + file_path: str, + position: Tuple[float, float, float], + resolution: Tuple[int, int] = (1024, 1024), + output_path: str = "polaris_view.png", + distance_factor: float = 1.0, + show_bounds: bool = False, + return_png: bool = False, + ) -> Union[str, bytes]: + """ + Render a view from a specified position in the Polaris system, + with camera direction calculated as position-to-centroid vector. + Args: + file_path: Path to the .glb file + position: Camera position in the Polaris system + resolution: Tuple of (width, height) for the output image + output_path: Path to save the rendered image + distance_factor: Factor to multiply the bounding box diagonal length by to determine camera distance + show_bounds: Whether to show bounding box + return_png: If True, return the PNG data instead of saving to file + Returns: + Path to the saved image or PNG data as bytes if return_png=True + """ + try: + scene = ModelLoader.load_from_glb(file_path) + bbox = BoundingBox(scene) + if show_bounds: + scene = bbox.add_to_scene() + centroid = scene.centroid + diagonal_length = np.linalg.norm(bbox.max_bound - bbox.min_bound) + direction_vector = np.array(position) - np.array(centroid) + direction_norm = np.linalg.norm(direction_vector) + if direction_norm > 0: + normalized_direction = direction_vector / direction_norm + adjusted_distance = diagonal_length * distance_factor + adjusted_position = (np.array(centroid) + normalized_direction * adjusted_distance) + camera_position = tuple(adjusted_position) + direction = tuple(normalized_direction) + else: + camera_position = position + direction = tuple(direction_vector) + renderer = SceneRenderer(scene) + result = renderer.render_from_position_and_direction( + camera_position, + direction, + resolution, + output_path, + return_png=return_png, + ) + if not return_png: + print( + f"Image saved to {output_path} with distance factor {distance_factor} (diagonal: {diagonal_length:.2f})" + ) + return result + except Exception as e: + print(f"Error rendering from Polaris position: {str(e)}") + raise + + @staticmethod + def render_six_views_polaris( + file_path: str, + resolution: Tuple[int, int] = (1024, 1024), + output_prefix: str = "polaris_view", + distance_factor: float = 1.0, + show_bounds: bool = False, + return_paths: bool = True, + ) -> Union[List[str], List[bytes]]: + """ + Render six orthogonal views using the polaris position approach. + Args: + file_path: Path to the .glb file + resolution: Tuple of (width, height) for the output images + output_prefix: Prefix for output image filenames + distance_factor: Factor to multiply the bounding box diagonal length to determine camera distance + show_bounds: Whether to show bounding box + return_paths: If True, return file paths, otherwise return in-memory PNG data + Returns: + List of paths to the saved images or list of PNG data as bytes if return_paths=False + """ + try: + scene = ModelLoader.load_from_glb(file_path) + bbox = BoundingBox(scene) + face_centers = bbox.calculate_face_centers() + direction_names = ["front", "back", "left", "right", "bottom", "top"] + results = [] + for i, position in enumerate(face_centers): + image_path = f"{output_prefix}_{direction_names[i]}.png" + result = GLBRenderer.render_from_polaris_position( + file_path, + position, + resolution, + image_path, + distance_factor, + show_bounds, + return_png=not return_paths, + ) + results.append(result) + return results + except Exception as e: + print(f"Error rendering six views with polaris: {str(e)}") + raise + + +def rotate_camera_positions(positions: List[Tuple[float, float, float]], + centroid: Tuple[float, float, float]) -> List[Tuple[float, float, float]]: + """ + Rotate a set of camera positions around the centroid by a random angle between 10-30 degrees. + Args: + positions: List of camera positions + centroid: Center point to rotate around + Returns: + List of rotated camera positions + """ + angle_x = np.radians(random.uniform(10, 30)) + angle_y = angle_x + angle_z = angle_x + rotation_x = np.array([ + [1, 0, 0], + [0, np.cos(angle_x), -np.sin(angle_x)], + [0, np.sin(angle_x), np.cos(angle_x)], + ]) + rotation_y = np.array([ + [np.cos(angle_y), 0, np.sin(angle_y)], + [0, 1, 0], + [-np.sin(angle_y), 0, np.cos(angle_y)], + ]) + rotation_z = np.array([ + [np.cos(angle_z), -np.sin(angle_z), 0], + [np.sin(angle_z), np.cos(angle_z), 0], + [0, 0, 1], + ]) + rotation_matrix = np.dot(rotation_z, np.dot(rotation_y, rotation_x)) + rotated_positions = [] + for pos in positions: + pos_array = np.array(pos) + centroid_array = np.array(centroid) + rel_pos = pos_array - centroid_array + rotated_rel_pos = np.dot(rotation_matrix, rel_pos) + rotated_pos = rotated_rel_pos + centroid_array + rotated_positions.append(tuple(rotated_pos)) + return rotated_positions + + +def get_image_from_glb(glb_path: str) -> str: + """ + Generate six views from the GLB file, with the orthogonal camera framework rotated by a random angle, + and return a combined image as a single base64-encoded string. + Args: + glb_path: Path to the .glb file + standard_view_num: Ignored - always generates six views + rand_view_num: Ignored - no random views are generated + Returns: + Single base64-encoded PNG image as string containing all six views combined in a grid + """ + temp_dir = os.path.dirname(glb_path) + if not temp_dir: + temp_dir = "." + output_prefix = os.path.join(temp_dir, "temp_view") + try: + scene = ModelLoader.load_from_glb(glb_path) + bbox = BoundingBox(scene) + centroid = tuple(scene.centroid) + face_centers = bbox.calculate_face_centers() + rotated_positions = rotate_camera_positions(face_centers, centroid) + direction_names = ["front", "back", "left", "right", "bottom", "top"] + png_data_list = [] + for i, position in enumerate(rotated_positions): + png_data = GLBRenderer.render_from_polaris_position( + glb_path, + position=position, + resolution=(1024, 1024), + output_path=os.path.join(temp_dir, f"temp_view_{direction_names[i]}.png"), + distance_factor=1.0, + show_bounds=True, + return_png=True, + ) + png_data_list.append(png_data) + pil_images = [] + all_labels = direction_names + for png_data in png_data_list: + pil_images.append(PIL.Image.open(BytesIO(png_data))) + layout = (3, 2) + rows, cols = layout + img_width, img_height = pil_images[0].size + combined_width = cols * img_width + combined_height = rows * img_height + combined_img = PIL.Image.new("RGB", (combined_width, combined_height), color="white") + from PIL import ImageDraw, ImageFont + + draw = ImageDraw.Draw(combined_img) + try: + font = ImageFont.truetype("arial.ttf", size=int(img_height * 0.15)) + except IOError: + try: + font = ImageFont.truetype( + "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", + size=int(img_height * 0.075), + ) + except IOError: + font = ImageFont.load_default() + for i, (img, label) in enumerate(zip(pil_images, all_labels)): + row = i // cols + col = i % cols + x = col * img_width + y = row * img_height + combined_img.paste(img, (x, y)) + draw.text((x + 10, y + 10), label, fill=(0, 0, 0), font=font) + buffer = BytesIO() + combined_img.save(buffer, format="PNG") + buffer.seek(0) + combined_base64 = base64.b64encode(buffer.getvalue()).decode("utf-8") + return combined_base64 + except Exception as e: + print(f"Error in get_image_from_glb: {str(e)}") + return "" + + +def main(): + """Main function to parse arguments and call appropriate renderer.""" + + parser = argparse.ArgumentParser(description="Generate images from GLB files") + parser.add_argument("file_path", help="Path to the .glb file") + parser.add_argument("-s", "--six-views", action="store_true", help="Generate six orthogonal views") + parser.add_argument( + "-sr", + "--six-view-with-two-random", + action="store_true", + help="Generate six orthogonal views plus two random views", + ) + parser.add_argument( + "-sv", + "--standard-view-num", + type=int, + default=6, + help="Number of standard views to use (max 6)", + ) + parser.add_argument( + "-rv", + "--rand-view-num", + type=int, + default=2, + help="Number of random views to generate", + ) + parser.add_argument( + "-p", + "--polaris-position", + type=float, + nargs=3, + help="Render from a specific position (x y z) with direction towards centroid", + ) + parser.add_argument( + "-d", + "--distance-factor", + type=float, + default=1.0, + help="Distance factor to multiply bounding box diagonal length", + ) + parser.add_argument( + "-b", + "--show-bounds", + action="store_true", + help="Show bounding box in the rendered image", + ) + parser.add_argument( + "--resolution", + type=int, + nargs=2, + default=[1024, 1024], + help="Image resolution (width height)", + ) + parser.add_argument("--output", default=None, help="Output image path/prefix") + parser.add_argument( + "--in-memory", + action="store_true", + help="Generate in-memory images instead of saving to files", + ) + args = parser.parse_args() + try: + if args.polaris_position: + output_path = args.output or "polaris_view.png" + position = tuple(args.polaris_position) + result = GLBRenderer.render_from_polaris_position( + args.file_path, + position, + tuple(args.resolution), + output_path, + args.distance_factor, + args.show_bounds, + return_png=args.in_memory, + ) + if args.in_memory: + print(f"Generated in-memory image ({len(result)} bytes)") + elif (args.six_views or args.six_view_with_two_random or args.standard_view_num > 0 or args.rand_view_num > 0): + output_prefix = args.output or "polaris_view" + if args.six_view_with_two_random: + base64_image = get_image_from_glb(args.file_path) + elif args.six_views: + base64_image = get_image_from_glb(args.file_path) + else: + base64_image = get_image_from_glb( + args.file_path, + standard_view_num=args.standard_view_num, + rand_view_num=args.rand_view_num, + ) + if output_prefix: + combined_path = f"{output_prefix}_combined.png" + img_data = base64.b64decode(base64_image) + with open(combined_path, "wb") as f: + f.write(img_data) + print(f"Combined image saved to {combined_path}") + else: + print( + "Error: Please specify either --six-views (-s), --six-view-with-two-random (-sr), --standard-view-num (-sv), --rand-view-num (-rv), or --polaris-position (-p)" + ) + sys.exit(1) + except Exception as e: + print(f"Error: {str(e)}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/_GLOBAL_CONFIGS.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/_GLOBAL_CONFIGS.py new file mode 100644 index 00000000..6e29ab9a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/_GLOBAL_CONFIGS.py @@ -0,0 +1,40 @@ +# global configs +import os + +ROOT_PATH = os.path.abspath(__file__) +ROOT_PATH = ROOT_PATH[:ROOT_PATH.rfind("/")] +ROOT_PATH = ROOT_PATH[:ROOT_PATH.rfind("/") + 1] + +ASSETS_PATH = os.path.join(ROOT_PATH, "assets/") +EMBODIMENTS_PATH = os.path.join(ASSETS_PATH, "embodiments/") +TEXTURES_PATH = os.path.join(ASSETS_PATH, "background_texture/") +CONFIGS_PATH = os.path.join(ROOT_PATH, "task_config/") +SCRIPT_PATH = os.path.join(ROOT_PATH, "script/") +DESCRIPTION_PATH = os.path.join(ROOT_PATH, "description/") + +# Euler angles in world coordinates +# t3d.euler.quat2euler(quat) returns (theta_x, theta_y, theta_z) +# theta_y controls the pitch, and theta_z controls rotation around the axis perpendicular to the tabletop plane +GRASP_DIRECTION_DIC = { + "left": [0, 0, 0, -1], + "front_left": [-0.383, 0, 0, -0.924], + "front": [-0.707, 0, 0, -0.707], + "front_right": [-0.924, 0, 0, -0.383], + "right": [-1, 0, 0, 0], + "top_down": [-0.5, 0.5, -0.5, -0.5], + "down_right": [-0.707, 0, -0.707, 0], + "down_left": [0, 0.707, 0, -0.707], + "top_down_little_left": [-0.353523, 0.61239, -0.353524, -0.61239], + "top_down_little_right": [-0.61239, 0.353523, -0.61239, -0.353524], + "left_arm_perf": [-0.853532, 0.146484, -0.353542, -0.3536], + "right_arm_perf": [-0.353518, 0.353564, -0.14642, -0.853568], +} + +WORLD_DIRECTION_DIC = { + "left": [0, -0.707, 0, 0.707], # -z -y -x + "front": [0.5, -0.5, 0.5, 0.5], # y z -x + "right": [0.707, 0, 0.707, 0], # z y -x + "top_down": [0, 0.707, -0.707, 0], # -x -y -z +} + +ROTATE_NUM = 10 diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/__init__.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/__init__.py new file mode 100755 index 00000000..3c2370d2 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/__init__.py @@ -0,0 +1,2 @@ +from .utils import * +from ._GLOBAL_CONFIGS import * diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/_base_task.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/_base_task.py new file mode 100644 index 00000000..982e6ec5 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/_base_task.py @@ -0,0 +1,1723 @@ +import os +import re +import sapien.core as sapien +from sapien.render import clear_cache as sapien_clear_cache +from sapien.utils.viewer import Viewer +import numpy as np +import gymnasium as gym +import pdb +import toppra as ta +import json +import transforms3d as t3d +from collections import OrderedDict +import torch, random + +from .utils import * +import math +from .robot import Robot +from .camera import Camera + +from copy import deepcopy +import subprocess +from pathlib import Path +import trimesh +import imageio +import glob + + +from ._GLOBAL_CONFIGS import * + +from typing import Optional, Literal + +current_file_path = os.path.abspath(__file__) +parent_directory = os.path.dirname(current_file_path) + + +class Base_Task(gym.Env): + + def __init__(self): + pass + + # =========================================================== Init Task Env =========================================================== + def _init_task_env_(self, table_xy_bias=[0, 0], table_height_bias=0, **kwags): + """ + Initialization TODO + - `self.FRAME_IDX`: The index of the file saved for the current scene. + - `self.fcitx5-configtool`: Left gripper pose (close <=0, open >=0.4). + - `self.ep_num`: Episode ID. + - `self.task_name`: Task name. + - `self.save_dir`: Save path.` + - `self.left_original_pose`: Left arm original pose. + - `self.right_original_pose`: Right arm original pose. + - `self.left_arm_joint_id`: [6,14,18,22,26,30]. + - `self.right_arm_joint_id`: [7,15,19,23,27,31]. + - `self.render_fre`: Render frequency. + """ + super().__init__() + ta.setup_logging("CRITICAL") # hide logging + np.random.seed(kwags.get("seed", 0)) + torch.manual_seed(kwags.get("seed", 0)) + # random.seed(kwags.get('seed', 0)) + + self.FRAME_IDX = 0 + self.task_name = kwags.get("task_name") + self.save_dir = kwags.get("save_path", "data") + self.ep_num = kwags.get("now_ep_num", 0) + self.render_freq = kwags.get("render_freq", 10) + self.data_type = kwags.get("data_type", None) + self.save_data = kwags.get("save_data", False) + self.dual_arm = kwags.get("dual_arm", True) + self.eval_mode = kwags.get("eval_mode", False) + + self.need_topp = True # TODO + + # Random + random_setting = kwags.get("domain_randomization") + self.random_background = random_setting.get("random_background", False) + self.cluttered_table = random_setting.get("cluttered_table", False) + self.clean_background_rate = random_setting.get("clean_background_rate", 1) + self.random_head_camera_dis = random_setting.get("random_head_camera_dis", 0) + self.random_table_height = random_setting.get("random_table_height", 0) + self.random_light = random_setting.get("random_light", False) + self.crazy_random_light_rate = random_setting.get("crazy_random_light_rate", 0) + self.crazy_random_light = (0 if not self.random_light else np.random.rand() < self.crazy_random_light_rate) + self.random_embodiment = random_setting.get("random_embodiment", False) # TODO + + self.file_path = [] + self.plan_success = True + self.step_lim = None + self.fix_gripper = False + self.setup_scene() + + self.left_js = None + self.right_js = None + self.raw_head_pcl = None + self.real_head_pcl = None + self.real_head_pcl_color = None + + self.now_obs = {} + self.take_action_cnt = 0 + self.eval_video_path = kwags.get("eval_video_save_dir", None) + + self.save_freq = kwags.get("save_freq") + self.world_pcd = None + + self.size_dict = list() + self.cluttered_objs = list() + self.prohibited_area = list() # [x_min, y_min, x_max, y_max] + self.record_cluttered_objects = list() # record cluttered objects info + + self.eval_success = False + self.table_z_bias = (np.random.uniform(low=-self.random_table_height, high=0) + table_height_bias) # TODO + self.need_plan = kwags.get("need_plan", True) + self.left_joint_path = kwags.get("left_joint_path", []) + self.right_joint_path = kwags.get("right_joint_path", []) + self.left_cnt = 0 + self.right_cnt = 0 + + self.instruction = None # for Eval + + self.create_table_and_wall(table_xy_bias=table_xy_bias, table_height=0.74) + self.load_robot(**kwags) + self.load_camera(**kwags) + self.robot.move_to_homestate() + + render_freq = self.render_freq + self.render_freq = 0 + self.together_open_gripper(save_freq=None) + self.render_freq = render_freq + + self.robot.set_origin_endpose() + self.load_actors() + + if self.cluttered_table: + self.get_cluttered_table() + + is_stable, unstable_list = self.check_stable() + if not is_stable: + raise UnStableError( + f'Objects is unstable in seed({kwags.get("seed", 0)}), unstable objects: {", ".join(unstable_list)}') + + if self.eval_mode: + with open(os.path.join(CONFIGS_PATH, "_eval_step_limit.yml"), "r") as f: + try: + data = yaml.safe_load(f) + self.step_lim = data[self.task_name] + except: + print(f"{self.task_name} not in step limit file, set to 1000") + self.step_lim = 1000 + + # info + self.info = dict() + self.info["cluttered_table_info"] = self.record_cluttered_objects + self.info["texture_info"] = { + "wall_texture": self.wall_texture, + "table_texture": self.table_texture, + } + self.info["info"] = {} + + self.stage_success_tag = False + + def check_stable(self): + actors_list, actors_pose_list = [], [] + for actor in self.scene.get_all_actors(): + actors_list.append(actor) + + def get_sim(p1, p2): + return np.abs(cal_quat_dis(p1.q, p2.q) * 180) + + is_stable, unstable_list = True, [] + + def check(times): + nonlocal self, is_stable, actors_list, actors_pose_list + for _ in range(times): + self.scene.step() + for idx, actor in enumerate(actors_list): + actors_pose_list[idx].append(actor.get_pose()) + + for idx, actor in enumerate(actors_list): + final_pose = actors_pose_list[idx][-1] + for pose in actors_pose_list[idx][-200:]: + if get_sim(final_pose, pose) > 3.0: + is_stable = False + unstable_list.append(actor.get_name()) + break + + is_stable = True + for _ in range(2000): + self.scene.step() + for idx, actor in enumerate(actors_list): + actors_pose_list.append([actor.get_pose()]) + check(500) + return is_stable, unstable_list + + def play_once(self): + pass + + def check_success(self): + pass + + def setup_scene(self, **kwargs): + """ + Set the scene + - Set up the basic scene: light source, viewer. + """ + self.engine = sapien.Engine() + # declare sapien renderer + from sapien.render import set_global_config + + set_global_config(max_num_materials=50000, max_num_textures=50000) + self.renderer = sapien.SapienRenderer() + # give renderer to sapien sim + self.engine.set_renderer(self.renderer) + + sapien.render.set_camera_shader_dir("rt") + sapien.render.set_ray_tracing_samples_per_pixel(32) + sapien.render.set_ray_tracing_path_depth(8) + sapien.render.set_ray_tracing_denoiser("oidn") + + # declare sapien scene + scene_config = sapien.SceneConfig() + self.scene = self.engine.create_scene(scene_config) + # set simulation timestep + self.scene.set_timestep(kwargs.get("timestep", 1 / 250)) + # add ground to scene + self.scene.add_ground(kwargs.get("ground_height", 0)) + # set default physical material + self.scene.default_physical_material = self.scene.create_physical_material( + kwargs.get("static_friction", 0.5), + kwargs.get("dynamic_friction", 0.5), + kwargs.get("restitution", 0), + ) + # give some white ambient light of moderate intensity + self.scene.set_ambient_light(kwargs.get("ambient_light", [0.5, 0.5, 0.5])) + # default enable shadow unless specified otherwise + shadow = kwargs.get("shadow", True) + # default spotlight angle and intensity + direction_lights = kwargs.get("direction_lights", [[[0, 0.5, -1], [0.5, 0.5, 0.5]]]) + self.direction_light_lst = [] + for direction_light in direction_lights: + if self.random_light: + direction_light[1] = [ + np.random.rand(), + np.random.rand(), + np.random.rand(), + ] + self.direction_light_lst.append( + self.scene.add_directional_light(direction_light[0], direction_light[1], shadow=shadow)) + # default point lights position and intensity + point_lights = kwargs.get("point_lights", [[[1, 0, 1.8], [1, 1, 1]], [[-1, 0, 1.8], [1, 1, 1]]]) + self.point_light_lst = [] + for point_light in point_lights: + if self.random_light: + point_light[1] = [np.random.rand(), np.random.rand(), np.random.rand()] + self.point_light_lst.append(self.scene.add_point_light(point_light[0], point_light[1], shadow=shadow)) + + # initialize viewer with camera position and orientation + if self.render_freq: + self.viewer = Viewer(self.renderer) + self.viewer.set_scene(self.scene) + self.viewer.set_camera_xyz( + x=kwargs.get("camera_xyz_x", 0.4), + y=kwargs.get("camera_xyz_y", 0.22), + z=kwargs.get("camera_xyz_z", 1.5), + ) + self.viewer.set_camera_rpy( + r=kwargs.get("camera_rpy_r", 0), + p=kwargs.get("camera_rpy_p", -0.8), + y=kwargs.get("camera_rpy_y", 2.45), + ) + + def create_table_and_wall(self, table_xy_bias=[0, 0], table_height=0.74): + self.table_xy_bias = table_xy_bias + wall_texture, table_texture = None, None + table_height += self.table_z_bias + + if self.random_background: + texture_type = "seen" if not self.eval_mode else "unseen" + directory_path = f"./assets/background_texture/{texture_type}" + file_count = len( + [name for name in os.listdir(directory_path) if os.path.isfile(os.path.join(directory_path, name))]) + + # wall_texture, table_texture = random.randint(0, file_count - 1), random.randint(0, file_count - 1) + wall_texture, table_texture = np.random.randint(0, file_count), np.random.randint(0, file_count) + + self.wall_texture, self.table_texture = ( + f"{texture_type}/{wall_texture}", + f"{texture_type}/{table_texture}", + ) + if np.random.rand() <= self.clean_background_rate: + self.wall_texture = None + if np.random.rand() <= self.clean_background_rate: + self.table_texture = None + else: + self.wall_texture, self.table_texture = None, None + + self.wall = create_box( + self.scene, + sapien.Pose(p=[0, 1, 1.5]), + half_size=[3, 0.6, 1.5], + color=(1, 0.9, 0.9), + name="wall", + texture_id=self.wall_texture, + is_static=True, + ) + + self.table = create_table( + self.scene, + sapien.Pose(p=[table_xy_bias[0], table_xy_bias[1], table_height]), + length=1.2, + width=0.7, + height=table_height, + thickness=0.05, + is_static=True, + texture_id=self.table_texture, + ) + + def get_cluttered_table(self, cluttered_numbers=10, xlim=[-0.59, 0.59], ylim=[-0.34, 0.34], zlim=[0.741]): + self.record_cluttered_objects = [] # record cluttered objects + + xlim[0] += self.table_xy_bias[0] + xlim[1] += self.table_xy_bias[0] + ylim[0] += self.table_xy_bias[1] + ylim[1] += self.table_xy_bias[1] + + if np.random.rand() < self.clean_background_rate: + return + + task_objects_list = [] + for entity in self.scene.get_all_actors(): + actor_name = entity.get_name() + if actor_name == "": + continue + if actor_name in ["table", "wall", "ground"]: + continue + task_objects_list.append(actor_name) + self.obj_names, self.cluttered_item_info = get_available_cluttered_objects(task_objects_list) + + success_count = 0 + max_try = 50 + trys = 0 + + while success_count < cluttered_numbers and trys < max_try: + obj = np.random.randint(len(self.obj_names)) + obj_name = self.obj_names[obj] + obj_idx = np.random.randint(len(self.cluttered_item_info[obj_name]["ids"])) + obj_idx = self.cluttered_item_info[obj_name]["ids"][obj_idx] + obj_radius = self.cluttered_item_info[obj_name]["params"][obj_idx]["radius"] + obj_offset = self.cluttered_item_info[obj_name]["params"][obj_idx]["z_offset"] + obj_maxz = self.cluttered_item_info[obj_name]["params"][obj_idx]["z_max"] + + success, self.cluttered_obj = rand_create_cluttered_actor( + self.scene, + xlim=xlim, + ylim=ylim, + zlim=np.array(zlim) + self.table_z_bias, + modelname=obj_name, + modelid=obj_idx, + modeltype=self.cluttered_item_info[obj_name]["type"], + rotate_rand=True, + rotate_lim=[0, 0, math.pi], + size_dict=self.size_dict, + obj_radius=obj_radius, + z_offset=obj_offset, + z_max=obj_maxz, + prohibited_area=self.prohibited_area, + ) + if not success or self.cluttered_obj is None: + trys += 1 + continue + self.cluttered_obj.set_name(f"{obj_name}") + self.cluttered_objs.append(self.cluttered_obj) + pose = self.cluttered_obj.get_pose().p.tolist() + pose.append(obj_radius) + self.size_dict.append(pose) + success_count += 1 + self.record_cluttered_objects.append({"object_type": obj_name, "object_index": obj_idx}) + + if success_count < cluttered_numbers: + print(f"Warning: Only {success_count} cluttered objects are placed on the table.") + + self.size_dict = None + self.cluttered_objs = [] + + def load_robot(self, **kwags): + """ + load aloha robot urdf file, set root pose and set joints + """ + if not hasattr(self, "robot"): + self.robot = Robot(self.scene, self.need_topp, **kwags) + self.robot.set_planner(self.scene) + self.robot.init_joints() + else: + self.robot.reset(self.scene, self.need_topp, **kwags) + + for link in self.robot.left_entity.get_links(): + link: sapien.physx.PhysxArticulationLinkComponent = link + link.set_mass(1) + for link in self.robot.right_entity.get_links(): + link: sapien.physx.PhysxArticulationLinkComponent = link + link.set_mass(1) + + def load_camera(self, **kwags): + """ + Add cameras and set camera parameters + - Including four cameras: left, right, front, head. + """ + + self.cameras = Camera( + bias=self.table_z_bias, + random_head_camera_dis=self.random_head_camera_dis, + **kwags, + ) + self.cameras.load_camera(self.scene) + self.scene.step() # run a physical step + self.scene.update_render() # sync pose from SAPIEN to renderer + + # =========================================================== Sapien =========================================================== + + def _update_render(self): + """ + Update rendering to refresh the camera's RGBD information + (rendering must be updated even when disabled, otherwise data cannot be collected). + """ + if self.crazy_random_light: + for renderColor in self.point_light_lst: + renderColor.set_color([np.random.rand(), np.random.rand(), np.random.rand()]) + for renderColor in self.direction_light_lst: + renderColor.set_color([np.random.rand(), np.random.rand(), np.random.rand()]) + now_ambient_light = self.scene.ambient_light + now_ambient_light = np.clip(np.array(now_ambient_light) + np.random.rand(3) * 0.2 - 0.1, 0, 1) + self.scene.set_ambient_light(now_ambient_light) + self.cameras.update_wrist_camera(self.robot.left_camera.get_pose(), self.robot.right_camera.get_pose()) + self.scene.update_render() + + # =========================================================== Basic APIs =========================================================== + + def get_obs(self): + self._update_render() + self.cameras.update_picture() + pkl_dic = { + "observation": {}, + "pointcloud": [], + "joint_action": {}, + "endpose": {}, + } + + pkl_dic["observation"] = self.cameras.get_config() + # rgb + if self.data_type.get("rgb", False): + rgb = self.cameras.get_rgb() + for camera_name in rgb.keys(): + pkl_dic["observation"][camera_name].update(rgb[camera_name]) + + if self.data_type.get("third_view", False): + third_view_rgb = self.cameras.get_observer_rgb() + pkl_dic["third_view_rgb"] = third_view_rgb + # mesh_segmentation + if self.data_type.get("mesh_segmentation", False): + mesh_segmentation = self.cameras.get_segmentation(level="mesh") + for camera_name in mesh_segmentation.keys(): + pkl_dic["observation"][camera_name].update(mesh_segmentation[camera_name]) + # actor_segmentation + if self.data_type.get("actor_segmentation", False): + actor_segmentation = self.cameras.get_segmentation(level="actor") + for camera_name in actor_segmentation.keys(): + pkl_dic["observation"][camera_name].update(actor_segmentation[camera_name]) + # depth + if self.data_type.get("depth", False): + depth = self.cameras.get_depth() + for camera_name in depth.keys(): + pkl_dic["observation"][camera_name].update(depth[camera_name]) + # endpose + if self.data_type.get("endpose", False): + norm_gripper_val = [ + self.robot.get_left_gripper_val(), + self.robot.get_right_gripper_val(), + ] + left_endpose = self.get_arm_pose("left") + right_endpose = self.get_arm_pose("right") + pkl_dic["endpose"]["left_endpose"] = left_endpose + pkl_dic["endpose"]["left_gripper"] = norm_gripper_val[0] + pkl_dic["endpose"]["right_endpose"] = right_endpose + pkl_dic["endpose"]["right_gripper"] = norm_gripper_val[1] + # qpos + if self.data_type.get("qpos", False): + + left_jointstate = self.robot.get_left_arm_jointState() + right_jointstate = self.robot.get_right_arm_jointState() + + pkl_dic["joint_action"]["left_arm"] = left_jointstate[:-1] + pkl_dic["joint_action"]["left_gripper"] = left_jointstate[-1] + pkl_dic["joint_action"]["right_arm"] = right_jointstate[:-1] + pkl_dic["joint_action"]["right_gripper"] = right_jointstate[-1] + pkl_dic["joint_action"]["vector"] = np.array(left_jointstate + right_jointstate) + # pointcloud + if self.data_type.get("pointcloud", False): + pkl_dic["pointcloud"] = self.cameras.get_pcd(self.data_type.get("conbine", False)) + + self.now_obs = deepcopy(pkl_dic) + return pkl_dic + + def save_camera_rgb(self, save_path, camera_name='head_camera'): + self._update_render() + self.cameras.update_picture() + rgb = self.cameras.get_rgb() + save_img(save_path, rgb[camera_name]['rgb']) + + def _take_picture(self): # save data + if not self.save_data: + return + + print("saving: episode = ", self.ep_num, " index = ", self.FRAME_IDX, end="\r") + + if self.FRAME_IDX == 0: + self.folder_path = {"cache": f"{self.save_dir}/.cache/episode{self.ep_num}/"} + + for directory in self.folder_path.values(): # remove previous data + if os.path.exists(directory): + file_list = os.listdir(directory) + for file in file_list: + os.remove(directory + file) + + pkl_dic = self.get_obs() + save_pkl(self.folder_path["cache"] + f"{self.FRAME_IDX}.pkl", pkl_dic) # use cache + self.FRAME_IDX += 1 + + def save_traj_data(self, idx): + file_path = os.path.join(self.save_dir, "_traj_data", f"episode{idx}.pkl") + traj_data = { + "left_joint_path": deepcopy(self.left_joint_path), + "right_joint_path": deepcopy(self.right_joint_path), + } + save_pkl(file_path, traj_data) + + def load_tran_data(self, idx): + assert self.save_dir is not None, "self.save_dir is None" + file_path = os.path.join(self.save_dir, "_traj_data", f"episode{idx}.pkl") + with open(file_path, "rb") as f: + traj_data = pickle.load(f) + return traj_data + + def merge_pkl_to_hdf5_video(self): + if not self.save_data: + return + cache_path = self.folder_path["cache"] + target_file_path = f"{self.save_dir}/data/episode{self.ep_num}.hdf5" + target_video_path = f"{self.save_dir}/video/episode{self.ep_num}.mp4" + # print('Merging pkl to hdf5: ', cache_path, ' -> ', target_file_path) + + os.makedirs(f"{self.save_dir}/data", exist_ok=True) + process_folder_to_hdf5_video(cache_path, target_file_path, target_video_path) + + def remove_data_cache(self): + folder_path = self.folder_path["cache"] + GREEN = "\033[92m" + RED = "\033[91m" + RESET = "\033[0m" + try: + shutil.rmtree(folder_path) + print(f"{GREEN}Folder {folder_path} deleted successfully.{RESET}") + except OSError as e: + print(f"{RED}Error: {folder_path} is not empty or does not exist.{RESET}") + + def set_instruction(self, instruction=None): + self.instruction = instruction + + def get_instruction(self, instruction=None): + return self.instruction + + def set_path_lst(self, args): + self.need_plan = args.get("need_plan", True) + self.left_joint_path = args.get("left_joint_path", []) + self.right_joint_path = args.get("right_joint_path", []) + + def _set_eval_video_ffmpeg(self, ffmpeg): + self.eval_video_ffmpeg = ffmpeg + + def close_env(self, clear_cache=False): + if clear_cache: + # for actor in self.scene.get_all_actors(): + # self.scene.remove_actor(actor) + sapien_clear_cache() + self.close() + + def _del_eval_video_ffmpeg(self): + if self.eval_video_ffmpeg: + self.eval_video_ffmpeg.stdin.close() + self.eval_video_ffmpeg.wait() + del self.eval_video_ffmpeg + + def delay(self, delay_time, save_freq=None): + render_freq = self.render_freq + self.render_freq = 0 + + left_gripper_val = self.robot.get_left_gripper_val() + right_gripper_val = self.robot.get_right_gripper_val() + for i in range(delay_time): + self.together_close_gripper( + left_pos=left_gripper_val, + right_pos=right_gripper_val, + save_freq=save_freq, + ) + + self.render_freq = render_freq + + def set_gripper(self, set_tag="together", left_pos=None, right_pos=None): + """ + Set gripper posture + - `left_pos`: Left gripper pose + - `right_pos`: Right gripper pose + - `set_tag`: "left" to set the left gripper, "right" to set the right gripper, "together" to set both grippers simultaneously. + """ + alpha = 0.5 + + left_result, right_result = None, None + + if set_tag == "left" or set_tag == "together": + left_result = self.robot.left_plan_grippers(self.robot.get_left_gripper_val(), left_pos) + left_gripper_step = left_result["per_step"] + left_gripper_res = left_result["result"] + num_step = left_result["num_step"] + left_result["result"] = np.pad( + left_result["result"], + (0, int(alpha * num_step)), + mode="constant", + constant_values=left_gripper_res[-1], + ) # append + left_result["num_step"] += int(alpha * num_step) + if set_tag == "left": + return left_result + + if set_tag == "right" or set_tag == "together": + right_result = self.robot.right_plan_grippers(self.robot.get_right_gripper_val(), right_pos) + right_gripper_step = right_result["per_step"] + right_gripper_res = right_result["result"] + num_step = right_result["num_step"] + right_result["result"] = np.pad( + right_result["result"], + (0, int(alpha * num_step)), + mode="constant", + constant_values=right_gripper_res[-1], + ) # append + right_result["num_step"] += int(alpha * num_step) + if set_tag == "right": + return right_result + + return left_result, right_result + + def add_prohibit_area( + self, + actor: Actor | sapien.Entity | sapien.Pose | list | np.ndarray, + padding=0.01, + ): + + if (isinstance(actor, sapien.Pose) or isinstance(actor, list) or isinstance(actor, np.ndarray)): + actor_pose = transforms._toPose(actor) + actor_data = {} + else: + actor_pose = actor.get_pose() + if isinstance(actor, Actor): + actor_data = actor.config + else: + actor_data = {} + + scale: float = actor_data.get("scale", 1) + origin_bounding_size = (np.array(actor_data.get("extents", [0.1, 0.1, 0.1])) * scale / 2) + origin_bounding_pts = (np.array([ + [-1, -1, -1], + [-1, -1, 1], + [-1, 1, -1], + [-1, 1, 1], + [1, -1, -1], + [1, -1, 1], + [1, 1, -1], + [1, 1, 1], + ]) * origin_bounding_size) + + actor_matrix = actor_pose.to_transformation_matrix() + trans_bounding_pts = actor_matrix[:3, :3] @ origin_bounding_pts.T + actor_matrix[:3, 3].reshape(3, 1) + x_min = np.min(trans_bounding_pts[0]) - padding + x_max = np.max(trans_bounding_pts[0]) + padding + y_min = np.min(trans_bounding_pts[1]) - padding + y_max = np.max(trans_bounding_pts[1]) + padding + # add_robot_visual_box(self, [x_min, y_min, actor_matrix[3, 3]]) + # add_robot_visual_box(self, [x_max, y_max, actor_matrix[3, 3]]) + self.prohibited_area.append([x_min, y_min, x_max, y_max]) + + def is_left_gripper_open(self): + return self.robot.is_left_gripper_open() + + def is_right_gripper_open(self): + return self.robot.is_right_gripper_open() + + def is_left_gripper_open_half(self): + return self.robot.is_left_gripper_open_half() + + def is_right_gripper_open_half(self): + return self.robot.is_right_gripper_open_half() + + def is_left_gripper_close(self): + return self.robot.is_left_gripper_close() + + def is_right_gripper_close(self): + return self.robot.is_right_gripper_close() + + # =========================================================== Our APIS =========================================================== + + def together_close_gripper(self, save_freq=-1, left_pos=0, right_pos=0): + left_result, right_result = self.set_gripper(left_pos=left_pos, right_pos=right_pos, set_tag="together") + control_seq = { + "left_arm": None, + "left_gripper": left_result, + "right_arm": None, + "right_gripper": right_result, + } + self.take_dense_action(control_seq, save_freq=save_freq) + + def together_open_gripper(self, save_freq=-1, left_pos=1, right_pos=1): + left_result, right_result = self.set_gripper(left_pos=left_pos, right_pos=right_pos, set_tag="together") + control_seq = { + "left_arm": None, + "left_gripper": left_result, + "right_arm": None, + "right_gripper": right_result, + } + self.take_dense_action(control_seq, save_freq=save_freq) + + def left_move_to_pose( + self, + pose, + constraint_pose=None, + use_point_cloud=False, + use_attach=False, + save_freq=-1, + ): + """ + Interpolative planning with screw motion. + Will not avoid collision and will fail if the path contains collision. + """ + if not self.plan_success: + return + if pose is None: + self.plan_success = False + return + if type(pose) == sapien.Pose: + pose = pose.p.tolist() + pose.q.tolist() + + if self.need_plan: + left_result = self.robot.left_plan_path(pose, constraint_pose=constraint_pose) + self.left_joint_path.append(deepcopy(left_result)) + else: + left_result = deepcopy(self.left_joint_path[self.left_cnt]) + self.left_cnt += 1 + + if left_result["status"] != "Success": + self.plan_success = False + return + + return left_result + + def right_move_to_pose( + self, + pose, + constraint_pose=None, + use_point_cloud=False, + use_attach=False, + save_freq=-1, + ): + """ + Interpolative planning with screw motion. + Will not avoid collision and will fail if the path contains collision. + """ + if not self.plan_success: + return + if pose is None: + self.plan_success = False + return + if type(pose) == sapien.Pose: + pose = pose.p.tolist() + pose.q.tolist() + + if self.need_plan: + right_result = self.robot.right_plan_path(pose, constraint_pose=constraint_pose) + self.right_joint_path.append(deepcopy(right_result)) + else: + right_result = deepcopy(self.right_joint_path[self.right_cnt]) + self.right_cnt += 1 + + if right_result["status"] != "Success": + self.plan_success = False + return + + return right_result + + def together_move_to_pose( + self, + left_target_pose, + right_target_pose, + left_constraint_pose=None, + right_constraint_pose=None, + use_point_cloud=False, + use_attach=False, + save_freq=-1, + ): + """ + Interpolative planning with screw motion. + Will not avoid collision and will fail if the path contains collision. + """ + if not self.plan_success: + return + if left_target_pose is None or right_target_pose is None: + self.plan_success = False + return + if type(left_target_pose) == sapien.Pose: + left_target_pose = left_target_pose.p.tolist() + left_target_pose.q.tolist() + if type(right_target_pose) == sapien.Pose: + right_target_pose = (right_target_pose.p.tolist() + right_target_pose.q.tolist()) + save_freq = self.save_freq if save_freq == -1 else save_freq + if self.need_plan: + left_result = self.robot.left_plan_path(left_target_pose, constraint_pose=left_constraint_pose) + right_result = self.robot.right_plan_path(right_target_pose, constraint_pose=right_constraint_pose) + self.left_joint_path.append(deepcopy(left_result)) + self.right_joint_path.append(deepcopy(right_result)) + else: + left_result = deepcopy(self.left_joint_path[self.left_cnt]) + right_result = deepcopy(self.right_joint_path[self.right_cnt]) + self.left_cnt += 1 + self.right_cnt += 1 + + try: + left_success = left_result["status"] == "Success" + right_success = right_result["status"] == "Success" + if not left_success or not right_success: + self.plan_success = False + # return TODO + except Exception as e: + if left_result is None or right_result is None: + self.plan_success = False + return # TODO + + if save_freq != None: + self._take_picture() + + now_left_id = 0 + now_right_id = 0 + i = 0 + + left_n_step = left_result["position"].shape[0] if left_success else 0 + right_n_step = right_result["position"].shape[0] if right_success else 0 + + while now_left_id < left_n_step or now_right_id < right_n_step: + # set the joint positions and velocities for move group joints only. + # The others are not the responsibility of the planner + if (left_success and now_left_id < left_n_step + and (not right_success or now_left_id / left_n_step <= now_right_id / right_n_step)): + self.robot.set_arm_joints( + left_result["position"][now_left_id], + left_result["velocity"][now_left_id], + "left", + ) + now_left_id += 1 + + if (right_success and now_right_id < right_n_step + and (not left_success or now_right_id / right_n_step <= now_left_id / left_n_step)): + self.robot.set_arm_joints( + right_result["position"][now_right_id], + right_result["velocity"][now_right_id], + "right", + ) + now_right_id += 1 + + self.scene.step() + if self.render_freq and i % self.render_freq == 0: + self._update_render() + self.viewer.render() + + if save_freq != None and i % save_freq == 0: + self._update_render() + self._take_picture() + i += 1 + + if save_freq != None: + self._take_picture() + + def move( + self, + actions_by_arm1: tuple[ArmTag, list[Action]], + actions_by_arm2: tuple[ArmTag, list[Action]] = None, + save_freq=-1, + ): + """ + Take action for the robot. + """ + + def get_actions(actions, arm_tag: ArmTag) -> list[Action]: + if actions[1] is None: + if actions[0][0] == arm_tag: + return actions[0][1] + else: + return [] + else: + if actions[0][0] == actions[0][1]: + raise ValueError("") + if actions[0][0] == arm_tag: + return actions[0][1] + else: + return actions[1][1] + + if self.plan_success is False: + return False + + actions = [actions_by_arm1, actions_by_arm2] + left_actions = get_actions(actions, "left") + right_actions = get_actions(actions, "right") + + max_len = max(len(left_actions), len(right_actions)) + left_actions += [None] * (max_len - len(left_actions)) + right_actions += [None] * (max_len - len(right_actions)) + + for left, right in zip(left_actions, right_actions): + + if (left is not None and left.arm_tag != "left") or (right is not None + and right.arm_tag != "right"): # check + raise ValueError(f"Invalid arm tag: {left.arm_tag} or {right.arm_tag}. Must be 'left' or 'right'.") + + if (left is not None and left.action == "move") and (right is not None + and right.action == "move"): # together move + self.together_move_to_pose( # TODO + left_target_pose=left.target_pose, + right_target_pose=right.target_pose, + left_constraint_pose=left.args.get("constraint_pose"), + right_constraint_pose=right.args.get("constraint_pose"), + ) + if self.plan_success is False: + return False + continue # TODO + else: + control_seq = { + "left_arm": None, + "left_gripper": None, + "right_arm": None, + "right_gripper": None, + } + if left is not None: + if left.action == "move": + control_seq["left_arm"] = self.left_move_to_pose( + pose=left.target_pose, + constraint_pose=left.args.get("constraint_pose"), + ) + else: # left.action == 'gripper' + control_seq["left_gripper"] = self.set_gripper(left_pos=left.target_gripper_pos, set_tag="left") + if self.plan_success is False: + return False + + if right is not None: + if right.action == "move": + control_seq["right_arm"] = self.right_move_to_pose( + pose=right.target_pose, + constraint_pose=right.args.get("constraint_pose"), + ) + else: # right.action == 'gripper' + control_seq["right_gripper"] = self.set_gripper(right_pos=right.target_gripper_pos, + set_tag="right") + if self.plan_success is False: + return False + + self.take_dense_action(control_seq) + + return True + + def get_gripper_actor_contact_position(self, actor_name): + contacts = self.scene.get_contacts() + position_lst = [] + for contact in contacts: + if (contact.bodies[0].entity.name == actor_name or contact.bodies[1].entity.name == actor_name): + contact_object = (contact.bodies[1].entity.name + if contact.bodies[0].entity.name == actor_name else contact.bodies[0].entity.name) + if contact_object in self.robot.gripper_name: + for point in contact.points: + position_lst.append(point.position) + return position_lst + + def check_actors_contact(self, actor1, actor2): + """ + Check if two actors are in contact. + - actor1: The first actor. + - actor2: The second actor. + """ + contacts = self.scene.get_contacts() + for contact in contacts: + if (contact.bodies[0].entity.name == actor1 + and contact.bodies[1].entity.name == actor2) or (contact.bodies[0].entity.name == actor2 + and contact.bodies[1].entity.name == actor1): + return True + return False + + def get_scene_contact(self): + contacts = self.scene.get_contacts() + for contact in contacts: + pdb.set_trace() + print(dir(contact)) + print(contact.bodies[0].entity.name, contact.bodies[1].entity.name) + + def choose_best_pose(self, res_pose, center_pose, arm_tag: ArmTag = None): + """ + Choose the best pose from the list of target poses. + - target_lst: List of target poses. + """ + if not self.plan_success: + return [-1, -1, -1, -1, -1, -1, -1] + if arm_tag == "left": + plan_multi_pose = self.robot.left_plan_multi_path + elif arm_tag == "right": + plan_multi_pose = self.robot.right_plan_multi_path + target_lst = self.robot.create_target_pose_list(res_pose, center_pose, arm_tag) + pose_num = len(target_lst) + traj_lst = plan_multi_pose(target_lst) + now_pose = None + now_step = -1 + for i in range(pose_num): + if traj_lst["status"][i] != "Success": + continue + if now_pose is None or len(traj_lst["position"][i]) < now_step: + now_pose = target_lst[i] + return now_pose + + # test grasp pose of all contact points + def _print_all_grasp_pose_of_contact_points(self, actor: Actor, pre_dis: float = 0.1): + for i in range(len(actor.config["contact_points_pose"])): + print(i, self.get_grasp_pose(actor, pre_dis=pre_dis, contact_point_id=i)) + + def get_grasp_pose( + self, + actor: Actor, + arm_tag: ArmTag, + contact_point_id: int = 0, + pre_dis: float = 0.0, + ) -> list: + """ + Obtain the grasp pose through the marked grasp point. + - actor: The instance of the object to be grasped. + - arm_tag: The arm to be used, either "left" or "right". + - pre_dis: The distance in front of the grasp point. + - contact_point_id: The index of the grasp point. + """ + if not self.plan_success: + return [-1, -1, -1, -1, -1, -1, -1] + + contact_matrix = actor.get_contact_point(contact_point_id, "matrix") + if contact_matrix is None: + return None + global_contact_pose_matrix = contact_matrix @ np.array([[0, 0, 1, 0], [-1, 0, 0, 0], [0, -1, 0, 0], + [0, 0, 0, 1]]) + global_contact_pose_matrix_q = global_contact_pose_matrix[:3, :3] + global_grasp_pose_p = (global_contact_pose_matrix[:3, 3] + + global_contact_pose_matrix_q @ np.array([-0.12 - pre_dis, 0, 0]).T) + global_grasp_pose_q = t3d.quaternions.mat2quat(global_contact_pose_matrix_q) + res_pose = list(global_grasp_pose_p) + list(global_grasp_pose_q) + res_pose = self.choose_best_pose(res_pose, actor.get_contact_point(contact_point_id, "list"), arm_tag) + return res_pose + + def _default_choose_grasp_pose(self, actor: Actor, arm_tag: ArmTag, pre_dis: float) -> list: + """ + Default grasp pose function. + - actor: The target actor to be grasped. + - arm_tag: The arm to be used for grasping, either "left" or "right". + - pre_dis: The distance in front of the grasp point, default is 0.1. + """ + id = -1 + score = -1 + + for i, contact_point in actor.iter_contact_points("list"): + pose = self.get_grasp_pose(actor, arm_tag, pre_dis, i) + now_score = 0 + if not (contact_point[1] < -0.1 and pose[2] < 0.85 or contact_point[1] > 0.05 and pose[2] > 0.92): + now_score -= 1 + quat_dis = cal_quat_dis(pose[-4:], GRASP_DIRECTION_DIC[str(arm_tag) + "_arm_perf"]) + + return self.get_grasp_pose(actor, arm_tag, pre_dis=pre_dis) + + def choose_grasp_pose( + self, + actor: Actor, + arm_tag: ArmTag, + pre_dis=0.1, + target_dis=0, + contact_point_id: list | float = None, + ) -> list: + """ + Test the grasp pose function. + - actor: The actor to be grasped. + - arm_tag: The arm to be used for grasping, either "left" or "right". + - pre_dis: The distance in front of the grasp point, default is 0.1. + """ + if not self.plan_success: + return + res_pre_top_down_pose = None + res_top_down_pose = None + dis_top_down = 1e9 + res_pre_side_pose = None + res_side_pose = None + dis_side = 1e9 + res_pre_pose = None + res_pose = None + dis = 1e9 + + pref_direction = self.robot.get_grasp_perfect_direction(arm_tag) + + def get_grasp_pose(pre_grasp_pose, pre_grasp_dis): + grasp_pose = deepcopy(pre_grasp_pose) + grasp_pose = np.array(grasp_pose) + direction_mat = t3d.quaternions.quat2mat(grasp_pose[-4:]) + grasp_pose[:3] += [pre_grasp_dis, 0, 0] @ np.linalg.inv(direction_mat) + grasp_pose = grasp_pose.tolist() + return grasp_pose + + def check_pose(pre_pose, pose, arm_tag): + if arm_tag == "left": + plan_func = self.robot.left_plan_path + else: + plan_func = self.robot.right_plan_path + pre_path = plan_func(pre_pose) + if pre_path["status"] != "Success": + return False + pre_qpos = pre_path["position"][-1] + return plan_func(pose)["status"] == "Success" + + if contact_point_id is not None: + if type(contact_point_id) != list: + contact_point_id = [contact_point_id] + contact_point_id = [(i, None) for i in contact_point_id] + else: + contact_point_id = actor.iter_contact_points() + + for i, _ in contact_point_id: + pre_pose = self.get_grasp_pose(actor, arm_tag, contact_point_id=i, pre_dis=pre_dis) + if pre_pose is None: + continue + pose = get_grasp_pose(pre_pose, pre_dis - target_dis) + now_dis_top_down = cal_quat_dis( + pose[-4:], + GRASP_DIRECTION_DIC[("top_down_little_left" if arm_tag == "right" else "top_down_little_right")], + ) + now_dis_side = cal_quat_dis(pose[-4:], GRASP_DIRECTION_DIC[pref_direction]) + + if res_pre_top_down_pose is None or now_dis_top_down < dis_top_down: + res_pre_top_down_pose = pre_pose + res_top_down_pose = pose + dis_top_down = now_dis_top_down + + if res_pre_side_pose is None or now_dis_side < dis_side: + res_pre_side_pose = pre_pose + res_side_pose = pose + dis_side = now_dis_side + + now_dis = 0.7 * now_dis_top_down + 0.3 * now_dis_side + if res_pre_pose is None or now_dis < dis: + res_pre_pose = pre_pose + res_pose = pose + dis = now_dis + + if dis_top_down < 0.15: + return res_pre_top_down_pose, res_top_down_pose + if dis_side < 0.15: + return res_pre_side_pose, res_side_pose + return res_pre_pose, res_pose + + def grasp_actor( + self, + actor: Actor, + arm_tag: ArmTag, + pre_grasp_dis=0.1, + grasp_dis=0, + gripper_pos=0.0, + contact_point_id: list | float = None, + ): + if not self.plan_success: + return None, [] + if self.need_plan == False: + if pre_grasp_dis == grasp_dis: + return arm_tag, [ + Action(arm_tag, "move", target_pose=[0, 0, 0, 0, 0, 0, 0]), + Action(arm_tag, "close", target_gripper_pos=gripper_pos), + ] + else: + return arm_tag, [ + Action(arm_tag, "move", target_pose=[0, 0, 0, 0, 0, 0, 0]), + Action( + arm_tag, + "move", + target_pose=[0, 0, 0, 0, 0, 0, 0], + constraint_pose=[1, 1, 1, 0, 0, 0], + ), + Action(arm_tag, "close", target_gripper_pos=gripper_pos), + ] + + pre_grasp_pose, grasp_pose = self.choose_grasp_pose( + actor, + arm_tag=arm_tag, + pre_dis=pre_grasp_dis, + target_dis=grasp_dis, + contact_point_id=contact_point_id, + ) + if pre_grasp_pose == grasp_pose: + return arm_tag, [ + Action(arm_tag, "move", target_pose=pre_grasp_pose), + Action(arm_tag, "close", target_gripper_pos=gripper_pos), + ] + else: + return arm_tag, [ + Action(arm_tag, "move", target_pose=pre_grasp_pose), + Action( + arm_tag, + "move", + target_pose=grasp_pose, + constraint_pose=[1, 1, 1, 0, 0, 0], + ), + Action(arm_tag, "close", target_gripper_pos=gripper_pos), + ] + + def get_place_pose( + self, + actor: Actor, + arm_tag: ArmTag, + target_pose: list | np.ndarray, + constrain: Literal["free", "align", "auto"] = "auto", + align_axis: list[np.ndarray] | np.ndarray | list = None, + actor_axis: np.ndarray | list = [1, 0, 0], + actor_axis_type: Literal["actor", "world"] = "actor", + functional_point_id: int = None, + pre_dis: float = 0.1, + pre_dis_axis: Literal["grasp", "fp"] | np.ndarray | list = "grasp", + ): + + if not self.plan_success: + return [-1, -1, -1, -1, -1, -1, -1] + + actor_matrix = actor.get_pose().to_transformation_matrix() + if functional_point_id is not None: + place_start_pose = actor.get_functional_point(functional_point_id, "pose") + z_transform = False + else: + place_start_pose = actor.get_pose() + z_transform = True + + end_effector_pose = (self.robot.get_left_ee_pose() if arm_tag == "left" else self.robot.get_right_ee_pose()) + + if constrain == "auto": + grasp_direct_vec = place_start_pose.p - end_effector_pose[:3] + if np.abs(np.dot(grasp_direct_vec, [0, 0, 1])) <= 0.1: + place_pose = get_place_pose( + place_start_pose, + target_pose, + constrain="align", + actor_axis=grasp_direct_vec, + actor_axis_type="world", + align_axis=[1, 1, 0] if arm_tag == "left" else [-1, 1, 0], + z_transform=z_transform, + ) + else: + camera_vec = transforms._toPose(end_effector_pose).to_transformation_matrix()[:3, 2] + place_pose = get_place_pose( + place_start_pose, + target_pose, + constrain="align", + actor_axis=camera_vec, + actor_axis_type="world", + align_axis=[0, 1, 0], + z_transform=z_transform, + ) + else: + place_pose = get_place_pose( + place_start_pose, + target_pose, + constrain=constrain, + actor_axis=actor_axis, + actor_axis_type=actor_axis_type, + align_axis=align_axis, + z_transform=z_transform, + ) + start2target = (transforms._toPose(place_pose).to_transformation_matrix()[:3, :3] + @ place_start_pose.to_transformation_matrix()[:3, :3].T) + target_point = (start2target @ (actor_matrix[:3, 3] - place_start_pose.p).reshape(3, 1)).reshape(3) + np.array( + place_pose[:3]) + + ee_pose_matrix = t3d.quaternions.quat2mat(end_effector_pose[-4:]) + target_grasp_matrix = start2target @ ee_pose_matrix + + res_matrix = np.eye(4) + res_matrix[:3, 3] = actor_matrix[:3, 3] - end_effector_pose[:3] + res_matrix[:3, 3] = np.linalg.inv(ee_pose_matrix) @ res_matrix[:3, 3] + target_grasp_qpose = t3d.quaternions.mat2quat(target_grasp_matrix) + + grasp_bias = target_grasp_matrix @ res_matrix[:3, 3] + if pre_dis_axis == "grasp": + target_dis_vec = target_grasp_matrix @ res_matrix[:3, 3] + target_dis_vec /= np.linalg.norm(target_dis_vec) + else: + target_pose_mat = transforms._toPose(target_pose).to_transformation_matrix() + if pre_dis_axis == "fp": + pre_dis_axis = [0.0, 0.0, 1.0] + pre_dis_axis = np.array(pre_dis_axis) + pre_dis_axis /= np.linalg.norm(pre_dis_axis) + target_dis_vec = (target_pose_mat[:3, :3] @ np.array(pre_dis_axis).reshape(3, 1)).reshape(3) + target_dis_vec /= np.linalg.norm(target_dis_vec) + res_pose = (target_point - grasp_bias - pre_dis * target_dis_vec).tolist() + target_grasp_qpose.tolist() + return res_pose + + def place_actor( + self, + actor: Actor, + arm_tag: ArmTag, + target_pose: list | np.ndarray, + functional_point_id: int = None, + pre_dis: float = 0.1, + dis: float = 0.02, + is_open: bool = True, + **args, + ): + if not self.plan_success: + return None, [] + if self.need_plan: + place_pre_pose = self.get_place_pose( + actor, + arm_tag, + target_pose, + functional_point_id=functional_point_id, + pre_dis=pre_dis, + **args, + ) + place_pose = self.get_place_pose( + actor, + arm_tag, + target_pose, + functional_point_id=functional_point_id, + pre_dis=dis, + **args, + ) + else: + place_pre_pose = [0, 0, 0, 0, 0, 0, 0] + place_pose = [0, 0, 0, 0, 0, 0, 0] + + actions = [ + Action(arm_tag, "move", target_pose=place_pre_pose), + Action(arm_tag, "move", target_pose=place_pose), + ] + if is_open: + actions.append(Action(arm_tag, "open", target_gripper_pos=1.0)) + return arm_tag, actions + + def move_by_displacement( + self, + arm_tag: ArmTag, + x: float = 0.0, + y: float = 0.0, + z: float = 0.0, + quat: list = None, + move_axis: Literal["world", "arm"] = "world", + ): + if arm_tag == "left": + origin_pose = np.array(self.robot.get_left_ee_pose(), dtype=np.float64) + elif arm_tag == "right": + origin_pose = np.array(self.robot.get_right_ee_pose(), dtype=np.float64) + else: + raise ValueError(f'arm_tag must be either "left" or "right", not {arm_tag}') + displacement = np.zeros(7, dtype=np.float64) + if move_axis == "world": + displacement[:3] = np.array([x, y, z], dtype=np.float64) + else: + dir_vec = transforms._toPose(origin_pose).to_transformation_matrix()[:3, 0] + dir_vec /= np.linalg.norm(dir_vec) + displacement[:3] = -z * dir_vec + origin_pose += displacement + if quat is not None: + origin_pose[3:] = quat + return arm_tag, [Action(arm_tag, "move", target_pose=origin_pose)] + + def move_to_pose( + self, + arm_tag: ArmTag, + target_pose: list | np.ndarray | sapien.Pose, + ): + return arm_tag, [Action(arm_tag, "move", target_pose=target_pose)] + + def close_gripper(self, arm_tag: ArmTag, pos: float = 0.0): + return arm_tag, [Action(arm_tag, "close", target_gripper_pos=pos)] + + def open_gripper(self, arm_tag: ArmTag, pos: float = 1.0): + return arm_tag, [Action(arm_tag, "open", target_gripper_pos=pos)] + + def back_to_origin(self, arm_tag: ArmTag): + if arm_tag == "left": + return arm_tag, [Action(arm_tag, "move", self.robot.left_original_pose)] + elif arm_tag == "right": + return arm_tag, [Action(arm_tag, "move", self.robot.right_original_pose)] + return None, [] + + def get_arm_pose(self, arm_tag: ArmTag): + if arm_tag == "left": + return self.robot.get_left_ee_pose() + elif arm_tag == "right": + return self.robot.get_right_ee_pose() + else: + raise ValueError(f'arm_tag must be either "left" or "right", not {arm_tag}') + + # =========================================================== Control Robot =========================================================== + + def take_dense_action(self, control_seq, save_freq=-1): + """ + control_seq: + left_arm, right_arm, left_gripper, right_gripper + """ + left_arm, left_gripper, right_arm, right_gripper = ( + control_seq["left_arm"], + control_seq["left_gripper"], + control_seq["right_arm"], + control_seq["right_gripper"], + ) + + save_freq = self.save_freq if save_freq == -1 else save_freq + if save_freq != None: + self._take_picture() + + max_control_len = 0 + + if left_arm is not None: + max_control_len = max(max_control_len, left_arm["position"].shape[0]) + if left_gripper is not None: + max_control_len = max(max_control_len, left_gripper["num_step"]) + if right_arm is not None: + max_control_len = max(max_control_len, right_arm["position"].shape[0]) + if right_gripper is not None: + max_control_len = max(max_control_len, right_gripper["num_step"]) + + for control_idx in range(max_control_len): + + if (left_arm is not None and control_idx < left_arm["position"].shape[0]): # control left arm + self.robot.set_arm_joints( + left_arm["position"][control_idx], + left_arm["velocity"][control_idx], + "left", + ) + + if left_gripper is not None and control_idx < left_gripper["num_step"]: + self.robot.set_gripper( + left_gripper["result"][control_idx], + "left", + left_gripper["per_step"], + ) # TODO + + if (right_arm is not None and control_idx < right_arm["position"].shape[0]): # control right arm + self.robot.set_arm_joints( + right_arm["position"][control_idx], + right_arm["velocity"][control_idx], + "right", + ) + + if right_gripper is not None and control_idx < right_gripper["num_step"]: + self.robot.set_gripper( + right_gripper["result"][control_idx], + "right", + right_gripper["per_step"], + ) # TODO + + self.scene.step() + + if self.render_freq and control_idx % self.render_freq == 0: + self._update_render() + self.viewer.render() + + if save_freq != None and control_idx % save_freq == 0: + self._update_render() + self._take_picture() + + if save_freq != None: + self._take_picture() + + return True # TODO: maybe need try error + + def take_action(self, action, action_type:Literal['qpos', 'ee']='qpos'): # action_type: qpos or ee + if self.take_action_cnt == self.step_lim or self.eval_success: + return + + eval_video_freq = 1 # fixed + if (self.eval_video_path is not None and self.take_action_cnt % eval_video_freq == 0): + self.eval_video_ffmpeg.stdin.write(self.now_obs["observation"]["head_camera"]["rgb"].tobytes()) + + self.take_action_cnt += 1 + print(f"step: \033[92m{self.take_action_cnt} / {self.step_lim}\033[0m", end="\r") + + self._update_render() + if self.render_freq: + self.viewer.render() + + actions = np.array([action]) + left_jointstate = self.robot.get_left_arm_jointState() + right_jointstate = self.robot.get_right_arm_jointState() + left_arm_dim = len(left_jointstate) - 1 if action_type == 'qpos' else 7 + right_arm_dim = len(right_jointstate) - 1 if action_type == 'qpos' else 7 + current_jointstate = np.array(left_jointstate + right_jointstate) + + left_arm_actions, left_gripper_actions, left_current_qpos, left_path = ( + [], + [], + [], + [], + ) + right_arm_actions, right_gripper_actions, right_current_qpos, right_path = ( + [], + [], + [], + [], + ) + + left_arm_actions, left_gripper_actions = ( + actions[:, :left_arm_dim], + actions[:, left_arm_dim], + ) + right_arm_actions, right_gripper_actions = ( + actions[:, left_arm_dim + 1:left_arm_dim + right_arm_dim + 1], + actions[:, left_arm_dim + right_arm_dim + 1], + ) + left_current_gripper, right_current_gripper = ( + self.robot.get_left_gripper_val(), + self.robot.get_right_gripper_val(), + ) + + left_gripper_path = np.hstack((left_current_gripper, left_gripper_actions)) + right_gripper_path = np.hstack((right_current_gripper, right_gripper_actions)) + + if action_type == 'qpos': + left_current_qpos, right_current_qpos = ( + current_jointstate[:left_arm_dim], + current_jointstate[left_arm_dim + 1:left_arm_dim + right_arm_dim + 1], + ) + left_path = np.vstack((left_current_qpos, left_arm_actions)) + right_path = np.vstack((right_current_qpos, right_arm_actions)) + + # ========== TOPP ========== + # TODO + topp_left_flag, topp_right_flag = True, True + + try: + times, left_pos, left_vel, acc, duration = (self.robot.left_mplib_planner.TOPP(left_path, + 1 / 250, + verbose=True)) + left_result = dict() + left_result["position"], left_result["velocity"] = left_pos, left_vel + left_n_step = left_result["position"].shape[0] + except Exception as e: + # print("left arm TOPP error: ", e) + topp_left_flag = False + left_n_step = 50 # fixed + + if left_n_step == 0: + topp_left_flag = False + left_n_step = 50 # fixed + + try: + times, right_pos, right_vel, acc, duration = (self.robot.right_mplib_planner.TOPP(right_path, + 1 / 250, + verbose=True)) + right_result = dict() + right_result["position"], right_result["velocity"] = right_pos, right_vel + right_n_step = right_result["position"].shape[0] + except Exception as e: + # print("right arm TOPP error: ", e) + topp_right_flag = False + right_n_step = 50 # fixed + + if right_n_step == 0: + topp_right_flag = False + right_n_step = 50 # fixed + + elif action_type == 'ee': + + left_result = self.robot.left_plan_path(left_arm_actions[0]) + right_result = self.robot.right_plan_path(right_arm_actions[0]) + if left_result["status"] != "Success": + left_n_step = 50 + topp_left_flag = False + # print("left fail") + else: + left_n_step = left_result["position"].shape[0] + topp_left_flag = True + + if right_result["status"] != "Success": + right_n_step = 50 + topp_right_flag = False + # print("right fail") + else: + right_n_step = right_result["position"].shape[0] + topp_right_flag = True + + # ========== Gripper ========== + + left_mod_num = left_n_step % len(left_gripper_actions) + right_mod_num = right_n_step % len(right_gripper_actions) + left_gripper_step = [0] + [ + left_n_step // len(left_gripper_actions) + (1 if i < left_mod_num else 0) + for i in range(len(left_gripper_actions)) + ] + right_gripper_step = [0] + [ + right_n_step // len(right_gripper_actions) + (1 if i < right_mod_num else 0) + for i in range(len(right_gripper_actions)) + ] + + left_gripper = [] + for gripper_step in range(1, left_gripper_path.shape[0]): + region_left_gripper = np.linspace( + left_gripper_path[gripper_step - 1], + left_gripper_path[gripper_step], + left_gripper_step[gripper_step] + 1, + )[1:] + left_gripper = left_gripper + region_left_gripper.tolist() + left_gripper = np.array(left_gripper) + + right_gripper = [] + for gripper_step in range(1, right_gripper_path.shape[0]): + region_right_gripper = np.linspace( + right_gripper_path[gripper_step - 1], + right_gripper_path[gripper_step], + right_gripper_step[gripper_step] + 1, + )[1:] + right_gripper = right_gripper + region_right_gripper.tolist() + right_gripper = np.array(right_gripper) + + now_left_id, now_right_id = 0, 0 + + # ========== Control Loop ========== + while now_left_id < left_n_step or now_right_id < right_n_step: + + if (now_left_id < left_n_step and now_left_id / left_n_step <= now_right_id / right_n_step): + if topp_left_flag: + self.robot.set_arm_joints( + left_result["position"][now_left_id], + left_result["velocity"][now_left_id], + "left", + ) + self.robot.set_gripper(left_gripper[now_left_id], "left") + + now_left_id += 1 + + if (now_right_id < right_n_step and now_right_id / right_n_step <= now_left_id / left_n_step): + if topp_right_flag: + self.robot.set_arm_joints( + right_result["position"][now_right_id], + right_result["velocity"][now_right_id], + "right", + ) + self.robot.set_gripper(right_gripper[now_right_id], "right") + + now_right_id += 1 + + self.scene.step() + self._update_render() + + if self.check_success(): + self.eval_success = True + self.get_obs() # update obs + if (self.eval_video_path is not None): + self.eval_video_ffmpeg.stdin.write(self.now_obs["observation"]["head_camera"]["rgb"].tobytes()) + return + + self._update_render() + if self.render_freq: # UI + self.viewer.render() + + + def save_camera_images(self, task_name, step_name, generate_num_id, save_dir="./camera_images"): + """ + Save camera images - patched version to ensure consistent episode numbering across all steps. + + Args: + task_name (str): Name of the task. + step_name (str): Name of the step. + generate_num_id (int): Generated ID used to create subfolders under the task directory. + save_dir (str): Base directory to save images, default is './camera_images'. + + Returns: + dict: A dictionary containing image data from each camera. + """ + # print(f"Received generate_num_id in save_camera_images: {generate_num_id}") + + # Create a subdirectory specific to the task + task_dir = os.path.join(save_dir, task_name) + os.makedirs(task_dir, exist_ok=True) + + # Create a subdirectory for the given generate_num_id + generate_dir = os.path.join(task_dir, generate_num_id) + os.makedirs(generate_dir, exist_ok=True) + + obs = self.get_obs() + cam_obs = obs["observation"] + image_data = {} + + # Extract step number and description from step_name using regex + match = re.match(r'(step[_]?\d+)(?:_(.*))?', step_name) + if match: + step_num = match.group(1) + step_description = match.group(2) if match.group(2) else "" + else: + step_num = None + step_description = step_name + + # Only process head_camera + cam_name = "head_camera" + if cam_name in cam_obs: + rgb = cam_obs[cam_name]["rgb"] + if rgb.dtype != np.uint8: + rgb = (rgb * 255).clip(0, 255).astype(np.uint8) + + # Use the instance's ep_num as the episode number + episode_num = getattr(self, 'ep_num', 0) + + # Save image to the subdirectory for the specific generate_num_id + filename = f"episode{episode_num}_{step_num}_{step_description}.png" + filepath = os.path.join(generate_dir, filename) + imageio.imwrite(filepath, rgb) + image_data[cam_name] = rgb + + # print(f"Saving image with episode_num={episode_num}, filename: {filename}, path: {generate_dir}") + + return image_data diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/adjust_bottle.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/adjust_bottle.py new file mode 100644 index 00000000..180602ad --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/adjust_bottle.py @@ -0,0 +1,67 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math + + +class adjust_bottle(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + self.qpose_tag = np.random.randint(0, 2) + qposes = [[0.707, 0.0, 0.0, -0.707], [0.707, 0.0, 0.0, 0.707]] + xlims = [[-0.12, -0.08], [0.08, 0.12]] + + self.model_id = np.random.choice([13, 16]) + + self.bottle = rand_create_actor( + self, + xlim=xlims[self.qpose_tag], + ylim=[-0.13, -0.08], + zlim=[0.752], + rotate_rand=True, + qpos=qposes[self.qpose_tag], + modelname="001_bottle", + convex=True, + rotate_lim=(0, 0, 0.4), + model_id=self.model_id, + ) + self.delay(4) + self.add_prohibit_area(self.bottle, padding=0.15) + self.left_target_pose = [-0.25, -0.12, 0.95, 0, 1, 0, 0] + self.right_target_pose = [0.25, -0.12, 0.95, 0, 1, 0, 0] + + def play_once(self): + # Determine which arm to use based on qpose_tag (1 for right, else left) + arm_tag = ArmTag("right" if self.qpose_tag == 1 else "left") + # Select target pose based on qpose_tag (right_target_pose or left_target_pose) + target_pose = (self.right_target_pose if self.qpose_tag == 1 else self.left_target_pose) + + # Grasp the bottle with specified arm + self.move(self.grasp_actor(self.bottle, arm_tag=arm_tag, pre_grasp_dis=0.1)) + # Move the arm upward by 0.1 meters along z-axis + self.move(self.move_by_displacement(arm_tag=arm_tag, z=0.1, move_axis="arm")) + # Place the bottle at target pose (functional point 0) while keeping gripper closed + self.move( + self.place_actor( + self.bottle, + target_pose=target_pose, + arm_tag=arm_tag, + functional_point_id=0, + pre_dis=0.0, + is_open=False, + )) + + self.info["info"] = { + "{A}": f"001_bottle/base{self.model_id}", + "{a}": str(arm_tag), + } + return self.info + + def check_success(self): + target_hight = 0.9 + bottle_pose = self.bottle.get_functional_point(0) + return ((self.qpose_tag == 0 and bottle_pose[0] < -0.15) or + (self.qpose_tag == 1 and bottle_pose[0] > 0.15)) and bottle_pose[2] > target_hight diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/beat_block_hammer.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/beat_block_hammer.py new file mode 100644 index 00000000..d3b696cc --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/beat_block_hammer.py @@ -0,0 +1,87 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +from ._GLOBAL_CONFIGS import * + + +class beat_block_hammer(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + self.hammer = create_actor( + scene=self, + pose=sapien.Pose([0, -0.06, 0.783], [0, 0, 0.995, 0.105]), + modelname="020_hammer", + convex=True, + model_id=0, + ) + block_pose = rand_pose( + xlim=[-0.25, 0.25], + ylim=[-0.05, 0.15], + zlim=[0.76], + qpos=[1, 0, 0, 0], + rotate_rand=True, + rotate_lim=[0, 0, 0.5], + ) + while abs(block_pose.p[0]) < 0.05 or np.sum(pow(block_pose.p[:2], 2)) < 0.001: + block_pose = rand_pose( + xlim=[-0.25, 0.25], + ylim=[-0.05, 0.15], + zlim=[0.76], + qpos=[1, 0, 0, 0], + rotate_rand=True, + rotate_lim=[0, 0, 0.5], + ) + + self.block = create_box( + scene=self, + pose=block_pose, + half_size=(0.025, 0.025, 0.025), + color=(1, 0, 0), + name="box", + is_static=True, + ) + self.hammer.set_mass(0.001) + + self.add_prohibit_area(self.hammer, padding=0.10) + self.prohibited_area.append([ + block_pose.p[0] - 0.05, + block_pose.p[1] - 0.05, + block_pose.p[0] + 0.05, + block_pose.p[1] + 0.05, + ]) + + def play_once(self): + # Get the position of the block's functional point + block_pose = self.block.get_functional_point(0, "pose").p + # Determine which arm to use based on block position (left if block is on left side, else right) + arm_tag = ArmTag("left" if block_pose[0] < 0 else "right") + + # Grasp the hammer with the selected arm + self.move(self.grasp_actor(self.hammer, arm_tag=arm_tag, pre_grasp_dis=0.12, grasp_dis=0.01)) + # Move the hammer upwards + self.move(self.move_by_displacement(arm_tag, z=0.07, move_axis="arm")) + + # Place the hammer on the block's functional point (position 1) + self.move( + self.place_actor( + self.hammer, + target_pose=self.block.get_functional_point(1, "pose"), + arm_tag=arm_tag, + functional_point_id=0, + pre_dis=0.06, + dis=0, + is_open=False, + )) + + self.info["info"] = {"{A}": "020_hammer/base0", "{a}": str(arm_tag)} + return self.info + + def check_success(self): + hammer_target_pose = self.hammer.get_functional_point(0, "pose").p + block_pose = self.block.get_functional_point(1, "pose").p + eps = np.array([0.02, 0.02]) + return np.all(abs(hammer_target_pose[:2] - block_pose[:2]) < eps) and self.check_actors_contact( + self.hammer.get_name(), self.block.get_name()) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/blocks_ranking_rgb.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/blocks_ranking_rgb.py new file mode 100644 index 00000000..acc1d1bf --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/blocks_ranking_rgb.py @@ -0,0 +1,164 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math +import numpy as np + + +class blocks_ranking_rgb(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + while True: + block_pose_lst = [] + for i in range(3): + block_pose = rand_pose( + xlim=[-0.28, 0.28], + ylim=[-0.08, 0.05], + zlim=[0.765], + qpos=[1, 0, 0, 0], + ylim_prop=True, + rotate_rand=True, + rotate_lim=[0, 0, 0.75], + ) + + def check_block_pose(block_pose): + for j in range(len(block_pose_lst)): + if (np.sum(pow(block_pose.p[:2] - block_pose_lst[j].p[:2], 2)) < 0.01): + return False + return True + + while (abs(block_pose.p[0]) < 0.05 or np.sum(pow(block_pose.p[:2] - np.array([0, -0.1]), 2)) < 0.01 + or not check_block_pose(block_pose)): + block_pose = rand_pose( + xlim=[-0.28, 0.28], + ylim=[-0.08, 0.05], + zlim=[0.765], + qpos=[1, 0, 0, 0], + ylim_prop=True, + rotate_rand=True, + rotate_lim=[0, 0, 0.75], + ) + block_pose_lst.append(deepcopy(block_pose)) + eps = [0.12, 0.03] + block1_pose = block_pose_lst[0].p + block2_pose = block_pose_lst[1].p + block3_pose = block_pose_lst[2].p + if (np.all(abs(block1_pose[:2] - block2_pose[:2]) < eps) + and np.all(abs(block2_pose[:2] - block3_pose[:2]) < eps) and block1_pose[0] < block2_pose[0] + and block2_pose[0] < block3_pose[0]): + continue + else: + break + + size = np.random.uniform(0.015, 0.025) + half_size = (size, size, size) + self.block1 = create_box( + scene=self, + pose=block_pose_lst[0], + half_size=half_size, + color=(1, 0, 0), + name="box", + ) + self.block2 = create_box( + scene=self, + pose=block_pose_lst[1], + half_size=half_size, + color=(0, 1, 0), + name="box", + ) + self.block3 = create_box( + scene=self, + pose=block_pose_lst[2], + half_size=half_size, + color=(0, 0, 1), + name="box", + ) + + self.add_prohibit_area(self.block1, padding=0.05) + self.add_prohibit_area(self.block2, padding=0.05) + self.add_prohibit_area(self.block3, padding=0.05) + + self.prohibited_area.append([-0.17, -0.22, 0.17, -0.12]) + + # Generate random y position for all blocks + y_pose = np.random.uniform(-0.2, -0.1) + + # Define target poses for each block with random x positions + self.block1_target_pose = [ + np.random.uniform(-0.09, -0.08), + y_pose, + 0.74 + self.table_z_bias, + ] + [0, 1, 0, 0] + self.block2_target_pose = [ + np.random.uniform(-0.01, 0.01), + y_pose, + 0.74 + self.table_z_bias, + ] + [0, 1, 0, 0] + self.block3_target_pose = [ + np.random.uniform(0.08, 0.09), + y_pose, + 0.74 + self.table_z_bias, + ] + [0, 1, 0, 0] + + def play_once(self): + # Initialize last gripper state + self.last_gripper = None + + # Pick and place each block to their target positions + arm_tag1 = self.pick_and_place_block(self.block1, self.block1_target_pose) + arm_tag2 = self.pick_and_place_block(self.block2, self.block2_target_pose) + arm_tag3 = self.pick_and_place_block(self.block3, self.block3_target_pose) + + # Store information about the blocks and which arms were used + self.info["info"] = { + "{A}": "red block", + "{B}": "green block", + "{C}": "blue block", + "{a}": arm_tag1, + "{b}": arm_tag2, + "{c}": arm_tag3, + } + return self.info + + def pick_and_place_block(self, block, target_pose=None): + block_pose = block.get_pose().p + arm_tag = ArmTag("left" if block_pose[0] < 0 else "right") + + if self.last_gripper is not None and (self.last_gripper != arm_tag): + self.move( + self.grasp_actor(block, arm_tag=arm_tag, pre_grasp_dis=0.09, grasp_dis=0.01), # arm_tag + self.back_to_origin(arm_tag=arm_tag.opposite), # arm_tag.opposite + ) + else: + self.move(self.grasp_actor(block, arm_tag=arm_tag, pre_grasp_dis=0.09)) # arm_tag + + self.move(self.move_by_displacement(arm_tag=arm_tag, z=0.07)) # arm_tag + + self.move( + self.place_actor( + block, + target_pose=target_pose, + arm_tag=arm_tag, + functional_point_id=0, + pre_dis=0.09, + dis=0.02, + constrain="align", + )) + self.move(self.move_by_displacement(arm_tag=arm_tag, z=0.07, move_axis="arm")) # arm_tag + + self.last_gripper = arm_tag + return str(arm_tag) + + def check_success(self): + block1_pose = self.block1.get_pose().p + block2_pose = self.block2.get_pose().p + block3_pose = self.block3.get_pose().p + + eps = [0.13, 0.03] + + return (np.all(abs(block1_pose[:2] - block2_pose[:2]) < eps) + and np.all(abs(block2_pose[:2] - block3_pose[:2]) < eps) and block1_pose[0] < block2_pose[0] + and block2_pose[0] < block3_pose[0] and self.is_left_gripper_open() and self.is_right_gripper_open()) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/blocks_ranking_size.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/blocks_ranking_size.py new file mode 100644 index 00000000..366af581 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/blocks_ranking_size.py @@ -0,0 +1,158 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math +import numpy as np + + +class blocks_ranking_size(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + color_lst = [(np.random.random(), np.random.random(), np.random.random()) for i in range(3)] + halfsize_lst = [ + np.random.uniform(0.03, 0.033), + np.random.uniform(0.024, 0.027), + np.random.uniform(0.018, 0.021), + ] + while True: + block_pose_lst = [] + for i in range(3): + block_pose = rand_pose( + xlim=[-0.28, 0.28], + ylim=[-0.08, 0.05], + zlim=[0.741 + halfsize_lst[i]], + qpos=[1, 0, 0, 0], + ylim_prop=True, + rotate_rand=True, + rotate_lim=[0, 0, 0.75], + ) + + def check_block_pose(block_pose): + for j in range(len(block_pose_lst)): + if (np.sum(pow(block_pose.p[:2] - block_pose_lst[j].p[:2], 2)) < 0.01): + return False + return True + + while (abs(block_pose.p[0]) < 0.05 or np.sum(pow(block_pose.p[:2] - np.array([0, -0.1]), 2)) < 0.01 + or not check_block_pose(block_pose)): + block_pose = rand_pose( + xlim=[-0.28, 0.28], + ylim=[-0.08, 0.05], + zlim=[0.741 + halfsize_lst[i]], + qpos=[1, 0, 0, 0], + ylim_prop=True, + rotate_rand=True, + rotate_lim=[0, 0, 0.75], + ) + block_pose_lst.append(deepcopy(block_pose)) + eps = [0.12, 0.03] + block1_pose = block_pose_lst[0].p + block2_pose = block_pose_lst[1].p + block3_pose = block_pose_lst[2].p + if (np.all(abs(block1_pose[:2] - block2_pose[:2]) < eps) + and np.all(abs(block2_pose[:2] - block3_pose[:2]) < eps) and block1_pose[0] < block2_pose[0] + and block2_pose[0] < block3_pose[0]): + continue + else: + break + + def create_block(block_pose, size, color): + half_size = (size, size, size) + return create_box( + scene=self, + pose=block_pose, + half_size=half_size, + color=color, + name="box", + ) + + self.block1 = create_block(block_pose_lst[0], halfsize_lst[0], color_lst[0]) + self.block2 = create_block(block_pose_lst[1], halfsize_lst[1], color_lst[1]) + self.block3 = create_block(block_pose_lst[2], halfsize_lst[2], color_lst[2]) + + self.add_prohibit_area(self.block1, padding=0.1) + self.add_prohibit_area(self.block2, padding=0.1) + self.add_prohibit_area(self.block3, padding=0.1) + self.prohibited_area.append([-0.27, -0.22, 0.27, -0.12]) + + # Generate random y position for all blocks + y_pose = np.random.uniform(-0.2, -0.1) + + # Define target poses for each block with random x positions + self.block1_target_pose = [ + np.random.uniform(-0.1, -0.09), + y_pose, + 0.74 + self.table_z_bias, + ] + [0, 1, 0, 0] + self.block2_target_pose = [ + np.random.uniform(0.01, 0.02), + y_pose, + 0.74 + self.table_z_bias, + ] + [0, 1, 0, 0] + self.block3_target_pose = [ + np.random.uniform(0.08, 0.09), + y_pose, + 0.74 + self.table_z_bias, + ] + [0, 1, 0, 0] + + def play_once(self): + # Initialize last gripper state + self.last_gripper = None + + # Pick and place blocks in reverse order (3, 2, 1) + arm_tag3 = self.pick_and_place_block(self.block3, self.block3_target_pose) + arm_tag2 = self.pick_and_place_block(self.block2, self.block2_target_pose) + arm_tag1 = self.pick_and_place_block(self.block1, self.block1_target_pose) + + self.info["info"] = { + "{A}": "large block", + "{B}": "medium block", + "{C}": "small block", + "{a}": arm_tag1, + "{b}": arm_tag2, + "{c}": arm_tag3, + } + return self.info + + def pick_and_place_block(self, block, target_pose=None): + block_pose = block.get_pose().p + arm_tag = ArmTag("left" if block_pose[0] < 0 else "right") + + if self.last_gripper is not None and (self.last_gripper != arm_tag): + self.move( + self.grasp_actor(block, arm_tag=arm_tag, pre_grasp_dis=0.09), # arm_tag + self.back_to_origin(arm_tag=arm_tag.opposite), # arm_tag.opposite + ) + else: + self.move(self.grasp_actor(block, arm_tag=arm_tag, pre_grasp_dis=0.09)) # arm_tag + + self.move(self.move_by_displacement(arm_tag=arm_tag, z=0.07)) # arm_tag + + self.move( + self.place_actor( + block, + target_pose=target_pose, + arm_tag=arm_tag, + functional_point_id=0, + pre_dis=0.09, + dis=0.02, + constrain="align", + )) + self.move(self.move_by_displacement(arm_tag=arm_tag, z=0.07, move_axis="arm")) # arm_tag + + self.last_gripper = arm_tag + return str(arm_tag) + + def check_success(self): + block1_pose = self.block1.get_pose().p + block2_pose = self.block2.get_pose().p + block3_pose = self.block3.get_pose().p + + eps = [0.13, 0.03] + + return (np.all(abs(block1_pose[:2] - block2_pose[:2]) < eps) + and np.all(abs(block2_pose[:2] - block3_pose[:2]) < eps) and block1_pose[0] < block2_pose[0] + and block2_pose[0] < block3_pose[0] and self.is_left_gripper_open() and self.is_right_gripper_open()) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/camera/__init__.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/camera/__init__.py new file mode 100644 index 00000000..e1235c57 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/camera/__init__.py @@ -0,0 +1 @@ +from .camera import * diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/camera/camera.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/camera/camera.py new file mode 100644 index 00000000..46d960d5 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/camera/camera.py @@ -0,0 +1,583 @@ +import sapien.core as sapien +import numpy as np +import pdb +from PIL import Image, ImageColor +import open3d as o3d +import json +import transforms3d as t3d +import cv2 +import torch +import yaml +import trimesh +import math +from .._GLOBAL_CONFIGS import CONFIGS_PATH +import os +from sapien.sensor import StereoDepthSensor, StereoDepthSensorConfig + +try: + import pytorch3d.ops as torch3d_ops + + def fps(points, num_points=1024, use_cuda=True): + K = [num_points] + if use_cuda: + points = torch.from_numpy(points).cuda() + sampled_points, indices = torch3d_ops.sample_farthest_points(points=points.unsqueeze(0), K=K) + sampled_points = sampled_points.squeeze(0) + sampled_points = sampled_points.cpu().numpy() + else: + points = torch.from_numpy(points) + sampled_points, indices = torch3d_ops.sample_farthest_points(points=points.unsqueeze(0), K=K) + sampled_points = sampled_points.squeeze(0) + sampled_points = sampled_points.numpy() + + return sampled_points, indices + +except: + print("missing pytorch3d") + + def fps(points, num_points=1024, use_cuda=True): + print("fps error: missing pytorch3d") + exit() + + +class Camera: + + def __init__(self, bias=0, random_head_camera_dis=0, **kwags): + """ """ + self.pcd_crop = kwags.get("pcd_crop", False) + self.pcd_down_sample_num = kwags.get("pcd_down_sample_num", 0) + self.pcd_crop_bbox = kwags.get("bbox", [[-0.6, -0.35, 0.7401], [0.6, 0.35, 2]]) + self.pcd_crop_bbox[0][2] += bias + self.table_z_bias = bias + self.random_head_camera_dis = random_head_camera_dis + + self.static_camera_config = [] + self.head_camera_type = kwags["camera"].get("head_camera_type", "D435") + self.wrist_camera_type = kwags["camera"].get("wrist_camera_type", "D435") + + self.collect_head_camera = kwags["camera"].get("collect_head_camera", True) + self.collect_wrist_camera = kwags["camera"].get("collect_wrist_camera", True) + + # embodiment = kwags.get('embodiment') + # embodiment_config_path = os.path.join(CONFIGS_PATH, '_embodiment_config.yml') + # with open(embodiment_config_path, 'r', encoding='utf-8') as f: + # embodiment_types = yaml.load(f.read(), Loader=yaml.FullLoader) + # robot_file = embodiment_types[embodiment]['file_path'] + # if robot_file is None: + # raise "No embodiment files" + + # robot_config_file = os.path.join(robot_file, 'config.yml') + # with open(robot_config_file, 'r', encoding='utf-8') as f: + # embodiment_args = yaml.load(f.read(), Loader=yaml.FullLoader) + # TODO + self.static_camera_info_list = kwags["left_embodiment_config"]["static_camera_list"] + self.static_camera_num = len(self.static_camera_info_list) + + def load_camera(self, scene): + """ + Add cameras and set camera parameters + - Including four cameras: left, right, front, head. + """ + near, far = 0.1, 100 + camera_config_path = os.path.join(CONFIGS_PATH, "_camera_config.yml") + + assert os.path.isfile(camera_config_path), "task config file is missing" + + with open(camera_config_path, "r", encoding="utf-8") as f: + camera_args = yaml.load(f.read(), Loader=yaml.FullLoader) + + # sensor_mount_actor = scene.create_actor_builder().build_kinematic() + + # camera_args = get_camera_config() + def create_camera(camera_info, random_head_camera_dis=0): + if camera_info["type"] not in camera_args.keys(): + raise ValueError(f"Camera type {camera_info['type']} not supported") + + camera_config = camera_args[camera_info["type"]] + cam_pos = np.array(camera_info["position"]) + vector = np.random.randn(3) + random_dir = vector / np.linalg.norm(vector) + cam_pos = cam_pos + random_dir * np.random.uniform(low=0, high=random_head_camera_dis) + cam_forward = np.array(camera_info["forward"]) / np.linalg.norm(np.array(camera_info["forward"])) + cam_left = np.array(camera_info["left"]) / np.linalg.norm(np.array(camera_info["left"])) + up = np.cross(cam_forward, cam_left) + mat44 = np.eye(4) + mat44[:3, :3] = np.stack([cam_forward, cam_left, up], axis=1) + mat44[:3, 3] = cam_pos + + # ========================= sensor camera ========================= + # sensor_config = StereoDepthSensorConfig() + # sensor_config.rgb_resolution = (camera_config['w'], camera_config['h']) + + camera = scene.add_camera( + name=camera_info["name"], + width=camera_config["w"], + height=camera_config["h"], + fovy=np.deg2rad(camera_config["fovy"]), + near=near, + far=far, + ) + camera.entity.set_pose(sapien.Pose(mat44)) + + # ========================= sensor camera ========================= + # sensor_camera = StereoDepthSensor( + # sensor_config, + # sensor_mount_actor, + # sapien.Pose(mat44) + # ) + # camera.entity.set_pose(sapien.Pose(camera_info['position'])) + # return camera, sensor_camera, camera_config + return camera, camera_config + + # ================================= wrist camera ================================= + if self.collect_wrist_camera: + wrist_camera_config = camera_args[self.wrist_camera_type] + self.left_camera = scene.add_camera( + name="left_camera", + width=wrist_camera_config["w"], + height=wrist_camera_config["h"], + fovy=np.deg2rad(wrist_camera_config["fovy"]), + near=near, + far=far, + ) + + self.right_camera = scene.add_camera( + name="right_camera", + width=wrist_camera_config["w"], + height=wrist_camera_config["h"], + fovy=np.deg2rad(wrist_camera_config["fovy"]), + near=near, + far=far, + ) + + # ================================= sensor camera ================================= + # sensor_config = StereoDepthSensorConfig() + # sensor_config.rgb_resolution = (wrist_camera_config['w'], wrist_camera_config['h']) + # self.left_sensor_camera = StereoDepthSensor( + # sensor_config, + # sensor_mount_actor, + # sapien.Pose([0,0,0],[1,0,0,0]) + # ) + + # self.right_sensor_camera = StereoDepthSensor( + # sensor_config, + # sensor_mount_actor, + # sapien.Pose([0,0,0],[1,0,0,0]) + # ) + + # ================================= static camera ================================= + self.head_camera_id = None + self.static_camera_list = [] + # self.static_sensor_camera_list = [] + self.static_camera_name = [] + # static camera list + for i, camera_info in enumerate(self.static_camera_info_list): + if camera_info.get("forward") == None: + camera_info["forward"] = (-1 * np.array(camera_info["position"])).tolist() + if camera_info.get("left") == None: + camera_info["left"] = [ + -camera_info["forward"][1], + camera_info["forward"][0], + ] + [0] + + if camera_info["name"] == "head_camera": + if self.collect_head_camera: + self.head_camera_id = i + camera_info["type"] = self.head_camera_type + # camera, sensor_camera, camera_config = create_camera(camera_info) + camera, camera_config = create_camera(camera_info, + random_head_camera_dis=self.random_head_camera_dis) + self.static_camera_list.append(camera) + self.static_camera_name.append(camera_info["name"]) + # self.static_sensor_camera_list.append(sensor_camera) + self.static_camera_config.append(camera_config) + # ================================= sensor camera ================================= + # camera_config = get_camera_config(camera_info['type']) + # cam_pos = np.array(camera_info['position']) + # cam_forward = np.array(camera_info['forward']) / np.linalg.norm(np.array(camera_info['forward'])) + # cam_left = np.array(camera_info['left']) / np.linalg.norm(np.array(camera_info['left'])) + # up = np.cross(cam_forward, cam_left) + # mat44 = np.eye(4) + # mat44[:3, :3] = np.stack([cam_forward, cam_left, up], axis=1) + # mat44[:3, 3] = cam_pos + # sensor_config = StereoDepthSensorConfig() + # sensor_config.rgb_resolution = (camera_config['w'], camera_config['h']) + + # self.head_sensor = StereoDepthSensor( + # sensor_config, + # sensor_mount_actor, + # sapien.Pose(mat44) + # ) + else: + # camera, sensor_camera, camera_config = create_camera(camera_info) + camera, camera_config = create_camera(camera_info) + self.static_camera_list.append(camera) + self.static_camera_name.append(camera_info["name"]) + # self.static_sensor_camera_list.append(sensor_camera) + self.static_camera_config.append(camera_config) + + # observer camera + self.observer_camera = scene.add_camera( + name="observer_camera", + width=320, + height=240, + fovy=np.deg2rad(93), + near=near, + far=far, + ) + observer_cam_pos = np.array([0.0, 0.23, 1.33]) + observer_cam_forward = np.array([0, -1, -1.02]) + # observer_cam_left = np.array([1,-1, 0]) + observer_cam_left = np.array([1, 0, 0]) + observer_up = np.cross(observer_cam_forward, observer_cam_left) + observer_mat44 = np.eye(4) + observer_mat44[:3, :3] = np.stack([observer_cam_forward, observer_cam_left, observer_up], axis=1) + observer_mat44[:3, 3] = observer_cam_pos + self.observer_camera.entity.set_pose(sapien.Pose(observer_mat44)) + + # world pcd camera + self.world_camera1 = scene.add_camera( + name="world_camera1", + width=640, + height=480, + fovy=np.deg2rad(50), + near=near, + far=far, + ) + world_cam_pos = np.array([0.4, -0.4, 1.6]) + world_cam_forward = np.array([-1, 1, -1.4]) + world_cam_left = np.array([-1, -1, 0]) + world_cam_up = np.cross(world_cam_forward, world_cam_left) + world_cam_mat44 = np.eye(4) + world_cam_mat44[:3, :3] = np.stack([world_cam_forward, world_cam_left, world_cam_up], axis=1) + world_cam_mat44[:3, 3] = world_cam_pos + self.world_camera1.entity.set_pose(sapien.Pose(world_cam_mat44)) + + self.world_camera2 = scene.add_camera( + name="world_camera1", + width=640, + height=480, + fovy=np.deg2rad(50), + near=near, + far=far, + ) + world_cam_pos = np.array([-0.4, -0.4, 1.6]) + world_cam_forward = np.array([1, 1, -1.4]) + world_cam_left = np.array([-1, 1, 0]) + world_cam_up = np.cross(world_cam_forward, world_cam_left) + world_cam_mat44 = np.eye(4) + world_cam_mat44[:3, :3] = np.stack([world_cam_forward, world_cam_left, world_cam_up], axis=1) + world_cam_mat44[:3, 3] = world_cam_pos + self.world_camera2.entity.set_pose(sapien.Pose(world_cam_mat44)) + + def update_picture(self): + # camera + if self.collect_wrist_camera: + self.left_camera.take_picture() + self.right_camera.take_picture() + + for camera in self.static_camera_list: + camera.take_picture() + + # ================================= sensor camera ================================= + # self.head_sensor.take_picture() + # self.head_sensor.compute_depth() + + def update_wrist_camera(self, left_pose, right_pose): + """ + Update rendering to refresh the camera's RGBD information + (rendering must be updated even when disabled, otherwise data cannot be collected). + """ + if self.collect_wrist_camera: + self.left_camera.entity.set_pose(left_pose) + self.right_camera.entity.set_pose(right_pose) + + def get_config(self) -> dict: + res = {} + + def _get_config(camera): + camera_intrinsic_cv = camera.get_intrinsic_matrix() + camera_extrinsic_cv = camera.get_extrinsic_matrix() + camera_model_matrix = camera.get_model_matrix() + return { + "intrinsic_cv": camera_intrinsic_cv, + "extrinsic_cv": camera_extrinsic_cv, + "cam2world_gl": camera_model_matrix, + } + + if self.collect_wrist_camera: + res["left_camera"] = _get_config(self.left_camera) + res["right_camera"] = _get_config(self.right_camera) + + for camera, camera_name in zip(self.static_camera_list, self.static_camera_name): + if camera_name == "head_camera": + if self.collect_head_camera: + res[camera_name] = _get_config(camera) + else: + res[camera_name] = _get_config(camera) + # ================================= sensor camera ================================= + # res['head_sensor'] = res['head_camera'] + # print(res) + return res + + def get_rgb(self) -> dict: + rgba = self.get_rgba() + rgb = {} + for camera_name, camera_data in rgba.items(): + rgb[camera_name] = {} + rgb[camera_name]["rgb"] = camera_data["rgba"][:, :, :3] # Exclude alpha channel + return rgb + + # Get Camera RGBA + def get_rgba(self) -> dict: + + def _get_rgba(camera): + camera_rgba = camera.get_picture("Color") + camera_rgba_img = (camera_rgba * 255).clip(0, 255).astype("uint8") + return camera_rgba_img + + # ================================= sensor camera ================================= + # def _get_sensor_rgba(sensor): + # camera_rgba = sensor.get_rgb() + # camera_rgba_img = (camera_rgba * 255).clip(0, 255).astype("uint8")[:,:,:3] + # return camera_rgba_img + + res = {} + + if self.collect_wrist_camera: + res["left_camera"] = {} + res["right_camera"] = {} + res["left_camera"]["rgba"] = _get_rgba(self.left_camera) + res["right_camera"]["rgba"] = _get_rgba(self.right_camera) + + for camera, camera_name in zip(self.static_camera_list, self.static_camera_name): + if camera_name == "head_camera": + if self.collect_head_camera: + res[camera_name] = {} + res[camera_name]["rgba"] = _get_rgba(camera) + else: + res[camera_name] = {} + res[camera_name]["rgba"] = _get_rgba(camera) + # ================================= sensor camera ================================= + # res['head_sensor']['rgb'] = _get_sensor_rgba(self.head_sensor) + + return res + + def get_observer_rgb(self) -> dict: + self.observer_camera.take_picture() + + def _get_rgb(camera): + camera_rgba = camera.get_picture("Color") + camera_rgb_img = (camera_rgba * 255).clip(0, 255).astype("uint8")[:, :, :3] + return camera_rgb_img + + return _get_rgb(self.observer_camera) + + # Get Camera Segmentation + def get_segmentation(self, level="mesh") -> dict: + + def _get_segmentation(camera, level="mesh"): + # visual_id is the unique id of each visual shape + seg_labels = camera.get_picture("Segmentation") # [H, W, 4] + colormap = sorted(set(ImageColor.colormap.values())) + color_palette = np.array([ImageColor.getrgb(color) for color in colormap], dtype=np.uint8) + if level == "mesh": + label0_image = seg_labels[..., 0].astype(np.uint8) # mesh-level + elif level == "actor": + label0_image = seg_labels[..., 1].astype(np.uint8) # actor-level + return color_palette[label0_image] + + res = { + # 'left_camera':{}, + # 'right_camera':{} + } + + if self.collect_wrist_camera: + res["left_camera"] = {} + res["right_camera"] = {} + res["left_camera"][f"{level}_segmentation"] = _get_segmentation(self.left_camera, level=level) + res["right_camera"][f"{level}_segmentation"] = _get_segmentation(self.right_camera, level=level) + + for camera, camera_name in zip(self.static_camera_list, self.static_camera_name): + if camera_name == "head_camera": + if self.collect_head_camera: + res[camera_name] = {} + res[camera_name][f"{level}_segmentation"] = _get_segmentation(camera, level=level) + else: + res[camera_name] = {} + res[camera_name][f"{level}_segmentation"] = _get_segmentation(camera, level=level) + return res + + # Get Camera Depth + def get_depth(self) -> dict: + + def _get_depth(camera): + position = camera.get_picture("Position") + depth = -position[..., 2] + depth_image = (depth * 1000.0).astype(np.float64) + return depth_image + + def _get_sensor_depth(sensor): + depth = sensor.get_depth() + depth = (depth * 1000.0).astype(np.float64) + return depth + + res = {} + rgba = self.get_rgba() + + if self.collect_wrist_camera: + res["left_camera"] = {} + res["right_camera"] = {} + res["left_camera"]["depth"] = _get_depth(self.left_camera) + res["right_camera"]["depth"] = _get_depth(self.right_camera) + res["left_camera"]["depth"] *= rgba["left_camera"]["rgba"][:, :, 3] / 255 + res["right_camera"]["depth"] *= rgba["right_camera"]["rgba"][:, :, 3] / 255 + + for camera, camera_name in zip(self.static_camera_list, self.static_camera_name): + if camera_name == "head_camera": + if self.collect_head_camera: + res[camera_name] = {} + res[camera_name]["depth"] = _get_depth(camera) + res[camera_name]["depth"] *= rgba[camera_name]["rgba"][:, :, 3] / 255 + else: + res[camera_name] = {} + res[camera_name]["depth"] = _get_depth(camera) + res[camera_name]["depth"] *= rgba[camera_name]["rgba"][:, :, 3] / 255 + # res['head_sensor']['depth'] = _get_sensor_depth(self.head_sensor) + + return res + + # Get World PointCloud + def get_world_pcd(self): + self.world_camera1.take_picture() + self.world_camera2.take_picture() + + def _get_camera_pcd(camera, color=True): + rgba = camera.get_picture_cuda("Color").torch() # [H, W, 4] + position = camera.get_picture_cuda("Position").torch() + model_matrix = camera.get_model_matrix() + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model_matrix = torch.tensor(model_matrix, dtype=torch.float32).to(device) + + # Extract valid three-dimensional points and corresponding color data. + valid_mask = position[..., 3] < 1 + points_opengl = position[..., :3][valid_mask] + points_color = rgba[valid_mask][:, :3] + # Transform into the world coordinate system. + points_world = (torch.bmm( + points_opengl.view(1, -1, 3), + model_matrix[:3, :3].transpose(0, 1).view(-1, 3, 3), + ).squeeze(1) + model_matrix[:3, 3]) + + # Format color data. + points_color = torch.clamp(points_color, 0, 1) + points_world = points_world.squeeze(0) + + # Convert the tensor back to a NumPy array for use with Open3D. + points_world_np = points_world.cpu().numpy() + points_color_np = points_color.cpu().numpy() + # print(points_world_np.shape, points_color_np.shape) + + res_pcd = (np.hstack((points_world_np, points_color_np)) if color else points_world_np) + return res_pcd + + pcd1 = _get_camera_pcd(self.world_camera1, color=True) + pcd2 = _get_camera_pcd(self.world_camera2, color=True) + res_pcd = np.vstack((pcd1, pcd2)) + + return res_pcd + pcd_array, index = fps(res_pcd[:, :3], 2000) + index = index.detach().cpu().numpy()[0] + + return pcd_array + + # Get Camera PointCloud + def get_pcd(self, if_combine=False): + + def _get_camera_pcd(camera, point_num=0): + rgba = camera.get_picture_cuda("Color").torch() # [H, W, 4] + position = camera.get_picture_cuda("Position").torch() + model_matrix = camera.get_model_matrix() + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model_matrix = torch.tensor(model_matrix, dtype=torch.float32).to(device) + + # Extract valid three-dimensional points and corresponding color data. + valid_mask = position[..., 3] < 1 + points_opengl = position[..., :3][valid_mask] + points_color = rgba[valid_mask][:, :3] + # Transform into the world coordinate system. + points_world = (torch.bmm( + points_opengl.view(1, -1, 3), + model_matrix[:3, :3].transpose(0, 1).view(-1, 3, 3), + ).squeeze(1) + model_matrix[:3, 3]) + + # Format color data. + points_color = torch.clamp(points_color, 0, 1) + + points_world = points_world.squeeze(0) + + # If crop is needed + if self.pcd_crop: + min_bound = torch.tensor(self.pcd_crop_bbox[0], dtype=torch.float32).to(device) + max_bound = torch.tensor(self.pcd_crop_bbox[1], dtype=torch.float32).to(device) + inside_bounds_mask = (points_world.squeeze(0) >= min_bound).all(dim=1) & (points_world.squeeze(0) + <= max_bound).all(dim=1) + points_world = points_world[inside_bounds_mask] + points_color = points_color[inside_bounds_mask] + + # Convert the tensor back to a NumPy array for use with Open3D. + points_world_np = points_world.cpu().numpy() + points_color_np = points_color.cpu().numpy() + + if point_num > 0: + # points_world_np,index = fps(points_world_np,point_num) + index = index.detach().cpu().numpy()[0] + points_color_np = points_color_np[index, :] + + return np.hstack((points_world_np, points_color_np)) + + if self.head_camera_id is None: + print("No head camera in static camera list, pointcloud save error!") + return None + + combined_pcd = np.array([]) + + # Merge pointcloud + if if_combine: + # combined_pcd = np.vstack((head_pcd , left_pcd , right_pcd, front_pcd)) + if self.collect_wrist_camera: + combined_pcd = np.vstack(( + _get_camera_pcd(self.left_camera), + _get_camera_pcd(self.right_camera), + )) + for camera, camera_name in zip(self.static_camera_list, self.static_camera_name): + if camera_name == "head_camera": + if self.collect_head_camera: + combined_pcd = np.vstack((combined_pcd, _get_camera_pcd(camera))) + else: + combined_pcd = np.vstack((combined_pcd, _get_camera_pcd(camera))) + elif self.collect_head_camera: + combined_pcd = _get_camera_pcd(self.static_camera_list[self.head_camera_id]) + + def pad_array(numpy_array, target_num): + current_num = numpy_array.shape[0] + if current_num < target_num: + if current_num == 0: + numpy_array = np.zeros((target_num, 6)) + else: + pad_num = target_num - current_num + padding = np.zeros((pad_num, 6)) + numpy_array = np.vstack([numpy_array, padding]) + return numpy_array + + combined_pcd = pad_array(combined_pcd, self.pcd_down_sample_num) + + pcd_array, index = combined_pcd[:, :3], np.array(range(len(combined_pcd))) + + if self.pcd_down_sample_num > 0: + pcd_array, index = fps(combined_pcd[:, :3], self.pcd_down_sample_num) + index = index.detach().cpu().numpy()[0] + + return combined_pcd[index] diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/click_alarmclock.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/click_alarmclock.py new file mode 100644 index 00000000..03665bc7 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/click_alarmclock.py @@ -0,0 +1,92 @@ +from copy import deepcopy +from ._base_task import Base_Task +from .utils import * +import sapien +import math + + +class click_alarmclock(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + rand_pos = rand_pose( + xlim=[-0.25, 0.25], + ylim=[-0.2, 0.0], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, 3.14, 0], + ) + while abs(rand_pos.p[0]) < 0.05: + rand_pos = rand_pose( + xlim=[-0.25, 0.25], + ylim=[-0.2, 0.0], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, 3.14, 0], + ) + + self.alarmclock_id = np.random.choice([1, 3], 1)[0] + self.alarm = create_actor( + scene=self, + pose=rand_pos, + modelname="046_alarm-clock", + convex=True, + model_id=self.alarmclock_id, + is_static=True, + ) + self.add_prohibit_area(self.alarm, padding=0.05) + self.check_arm_function = self.is_left_gripper_close if self.alarm.get_pose().p[0] < 0 else self.is_right_gripper_close + + def play_once(self): + # Determine which arm to use based on alarm clock's position (right if positive x, left otherwise) + arm_tag = ArmTag("right" if self.alarm.get_pose().p[0] > 0 else "left") + + # Move the gripper above the top center of the alarm clock and close the gripper to simulate a click + # Note: although the code structure resembles a grasp, it is used here to simulate a touch/click action + # You can adjust API parameters to move above the top button and close the gripper (similar to grasp_actor) + self.move(( + ArmTag(arm_tag), + [ + Action( + arm_tag, + "move", + self.get_grasp_pose(self.alarm, pre_dis=0.1, contact_point_id=0, arm_tag=arm_tag)[:3] + + [0.5, -0.5, 0.5, 0.5], + ), + Action(arm_tag, "close", target_gripper_pos=0.0), + ], + )) + + # Move the gripper downward to press the top button of the alarm clock + self.move(self.move_by_displacement(arm_tag, z=-0.065)) + # Check whether the simulated click action was successful + self.check_success() + + # Move the gripper back to the original height (not lifting the alarm clock) + self.move(self.move_by_displacement(arm_tag, z=0.065)) + # Optionally check success again + self.check_success() + + # Record information about the alarm clock and the arm used + self.info["info"] = { + "{A}": f"046_alarm-clock/base{self.alarmclock_id}", + "{a}": str(arm_tag), + } + return self.info + + + def check_success(self): + if self.stage_success_tag: + return True + if not self.check_arm_function(): + return False + alarm_pose = self.alarm.get_contact_point(0)[:3] + positions = self.get_gripper_actor_contact_position("046_alarm-clock") + eps = [0.03, 0.03] + for position in positions: + if (np.all(np.abs(position[:2] - alarm_pose[:2]) < eps) and abs(position[2] - alarm_pose[2]) < 0.03): + self.stage_success_tag = True + return True + return False diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/click_bell.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/click_bell.py new file mode 100644 index 00000000..6782bf22 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/click_bell.py @@ -0,0 +1,83 @@ +from copy import deepcopy +from ._base_task import Base_Task +from .utils import * +import sapien +import math + + +class click_bell(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + rand_pos = rand_pose( + xlim=[-0.25, 0.25], + ylim=[-0.2, 0.0], + qpos=[0.5, 0.5, 0.5, 0.5], + ) + while abs(rand_pos.p[0]) < 0.05: + rand_pos = rand_pose( + xlim=[-0.25, 0.25], + ylim=[-0.2, 0.0], + qpos=[0.5, 0.5, 0.5, 0.5], + ) + + self.bell_id = np.random.choice([0, 1], 1)[0] + self.bell = create_actor( + scene=self, + pose=rand_pos, + modelname="050_bell", + convex=True, + model_id=self.bell_id, + is_static=True, + ) + + self.add_prohibit_area(self.bell, padding=0.07) + self.check_arm_function = self.is_left_gripper_close if self.bell.get_pose().p[0] < 0 else self.is_right_gripper_close + + def play_once(self): + # Choose the arm to use: right arm if the bell is on the right side (positive x), left otherwise + arm_tag = ArmTag("right" if self.bell.get_pose().p[0] > 0 else "left") + + # Move the gripper above the top center of the bell and close the gripper to simulate a click + # Note: grasp_actor here is not used to grasp the bell, but to simulate a touch/click action + # You must use the same pre_grasp_dis and grasp_dis values as in the click_bell task + self.move(self.grasp_actor( + self.bell, + arm_tag=arm_tag, + pre_grasp_dis=0.1, + grasp_dis=0.1, + contact_point_id=0, # Targeting the bell's top center + )) + + # Move the gripper downward to touch the top center of the bell + self.move(self.move_by_displacement(arm_tag, z=-0.045)) + + # Check whether the simulated click action was successful + self.check_success() + + # Move the gripper back up to the original position (no need to lift or grasp the bell) + self.move(self.move_by_displacement(arm_tag, z=0.045)) + + # Check success again if needed (optional, based on your task logic) + self.check_success() + + # Record which bell and arm were used in the info dictionary + self.info["info"] = {"{A}": f"050_bell/base{self.bell_id}", "{a}": str(arm_tag)} + return self.info + + + def check_success(self): + if self.stage_success_tag: + return True + if not self.check_arm_function(): + return False + bell_pose = self.bell.get_contact_point(0)[:3] + positions = self.get_gripper_actor_contact_position("050_bell") + eps = [0.025, 0.025] + for position in positions: + if (np.all(np.abs(position[:2] - bell_pose[:2]) < eps) and abs(position[2] - bell_pose[2]) < 0.03): + self.stage_success_tag = True + return True + return False diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/dump_bin_bigbin.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/dump_bin_bigbin.py new file mode 100644 index 00000000..2164e5b6 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/dump_bin_bigbin.py @@ -0,0 +1,162 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +from copy import deepcopy + + +class dump_bin_bigbin(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(table_xy_bias=[0.3, 0], **kwags) + + def load_actors(self): + self.dustbin = create_actor( + self, + pose=sapien.Pose([-0.45, 0, 0], [0.5, 0.5, 0.5, 0.5]), + modelname="011_dustbin", + convex=True, + is_static=True, + ) + deskbin_pose = rand_pose( + xlim=[-0.2, 0.2], + ylim=[-0.2, -0.05], + qpos=[0.651892, 0.651428, 0.274378, 0.274584], + rotate_rand=True, + rotate_lim=[0, np.pi / 8.5, 0], + ) + while abs(deskbin_pose.p[0]) < 0.05: + deskbin_pose = rand_pose( + xlim=[-0.2, 0.2], + ylim=[-0.2, -0.05], + qpos=[0.651892, 0.651428, 0.274378, 0.274584], + rotate_rand=True, + rotate_lim=[0, np.pi / 8.5, 0], + ) + + self.deskbin_id = np.random.choice([0, 3, 7, 8, 9, 10], 1)[0] + self.deskbin = create_actor( + self, + pose=deskbin_pose, + modelname="063_tabletrashbin", + model_id=self.deskbin_id, + convex=True, + ) + self.garbage_num = 5 + self.sphere_lst = [] + for i in range(self.garbage_num): + sphere_pose = sapien.Pose( + [ + deskbin_pose.p[0] + np.random.rand() * 0.02 - 0.01, + deskbin_pose.p[1] + np.random.rand() * 0.02 - 0.01, + 0.78 + i * 0.005, + ], + [1, 0, 0, 0], + ) + sphere = create_sphere( + self.scene, + pose=sphere_pose, + radius=0.008, + color=[1, 0, 0], + name="garbage", + ) + self.sphere_lst.append(sphere) + self.sphere_lst[-1].find_component_by_type(sapien.physx.PhysxRigidDynamicComponent).mass = 0.0001 + + self.add_prohibit_area(self.deskbin, padding=0.04) + self.prohibited_area.append([-0.2, -0.2, 0.2, 0.2]) + # Define target pose for placing + self.middle_pose = [0, -0.1, 0.741 + self.table_z_bias, 1, 0, 0, 0] + # Define movement actions for shaking the deskbin + action_lst = [ + Action( + ArmTag('left'), + "move", + [-0.45, -0.05, 1.05, -0.694654, -0.178228, 0.165979, -0.676862], + ), + Action( + ArmTag('left'), + "move", + [ + -0.45, + -0.05 - np.random.rand() * 0.02, + 1.05 - np.random.rand() * 0.02, + -0.694654, + -0.178228, + 0.165979, + -0.676862, + ], + ), + ] + self.pour_actions = (ArmTag('left'), action_lst) + + def play_once(self): + # Get deskbin's current position + deskbin_pose = self.deskbin.get_pose().p + # Determine which arm to use for grasping based on deskbin's position + grasp_deskbin_arm_tag = ArmTag("left" if deskbin_pose[0] < 0 else "right") + # Always use left arm for placing + place_deskbin_arm_tag = ArmTag("left") + + if grasp_deskbin_arm_tag == "right": + # Grasp the deskbin with right arm + self.move( + self.grasp_actor( + self.deskbin, + arm_tag=grasp_deskbin_arm_tag, + pre_grasp_dis=0.08, + contact_point_id=3, + )) + # Lift the deskbin up + self.move(self.move_by_displacement(grasp_deskbin_arm_tag, z=0.08, move_axis="arm")) + # Place the deskbin at target pose + self.move( + self.place_actor( + self.deskbin, + target_pose=self.middle_pose, + arm_tag=grasp_deskbin_arm_tag, + pre_dis=0.08, + dis=0.01, + )) + # Move arm up after placing + self.move(self.move_by_displacement(grasp_deskbin_arm_tag, z=0.1, move_axis="arm")) + # Return right arm to origin while simultaneously grasping with left arm + self.move( + self.back_to_origin(grasp_deskbin_arm_tag), + self.grasp_actor( + self.deskbin, + arm_tag=place_deskbin_arm_tag, + pre_grasp_dis=0.08, + contact_point_id=1, + ), + ) + else: + # If deskbin is on left side, directly grasp with left arm + self.move( + self.grasp_actor( + self.deskbin, + arm_tag=place_deskbin_arm_tag, + pre_grasp_dis=0.08, + contact_point_id=1, + )) + + # Lift the deskbin with left arm + self.move(self.move_by_displacement(arm_tag=place_deskbin_arm_tag, z=0.08, move_axis="arm")) + # Perform shaking motion 3 times + for i in range(3): + self.move(self.pour_actions) + # Delay for 6 seconds + self.delay(6) + + self.info["info"] = {"{A}": f"063_tabletrashbin/base{self.deskbin_id}"} + return self.info + + def check_success(self): + deskbin_pose = self.deskbin.get_pose().p + if deskbin_pose[2] < 1: + return False + for i in range(self.garbage_num): + pose = self.sphere_lst[i].get_pose().p + if pose[2] >= 0.13 and pose[2] <= 0.25: + continue + return False + return True diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/grab_roller.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/grab_roller.py new file mode 100644 index 00000000..eedea445 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/grab_roller.py @@ -0,0 +1,57 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math +from ._GLOBAL_CONFIGS import * +from copy import deepcopy + + +class grab_roller(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + ori_qpos = [[0.5, 0.5, 0.5, 0.5], [0.5, 0.5, 0.5, 0.5], [0, 0, 0.707, 0.707]] + self.model_id = np.random.choice([0, 2], 1)[0] + rand_pos = rand_pose( + xlim=[-0.15, 0.15], + ylim=[-0.25, -0.05], + qpos=ori_qpos[self.model_id], + rotate_rand=True, + rotate_lim=[0, 0.8, 0], + ) + self.roller = create_actor( + scene=self, + pose=rand_pos, + modelname="102_roller", + convex=True, + model_id=self.model_id, + ) + + self.add_prohibit_area(self.roller, padding=0.1) + + def play_once(self): + # Initialize arm tags for left and right arms + left_arm_tag = ArmTag("left") + right_arm_tag = ArmTag("right") + + # Grasp the roller with both arms simultaneously at different contact points + self.move( + self.grasp_actor(self.roller, left_arm_tag, pre_grasp_dis=0.08, contact_point_id=0), + self.grasp_actor(self.roller, right_arm_tag, pre_grasp_dis=0.08, contact_point_id=1), + ) + + # Lift the roller to height 0.85 by moving both arms upward simultaneously + self.move( + self.move_by_displacement(left_arm_tag, z=0.85 - self.roller.get_pose().p[2]), + self.move_by_displacement(right_arm_tag, z=0.85 - self.roller.get_pose().p[2]), + ) + + # Record information about the roller in the info dictionary + self.info["info"] = {"{A}": f"102_roller/base{self.model_id}"} + return self.info + + def check_success(self): + roller_pose = self.roller.get_pose().p + return (self.is_left_gripper_close() and self.is_right_gripper_close() and roller_pose[2] > 0.8) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/handover_block.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/handover_block.py new file mode 100644 index 00000000..219e3d53 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/handover_block.py @@ -0,0 +1,116 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math +from ._GLOBAL_CONFIGS import * + + +class handover_block(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + rand_pos = rand_pose( + xlim=[-0.25, -0.05], + ylim=[0, 0.25], + zlim=[0.842], + qpos=[0.981, 0, 0, 0.195], + rotate_rand=True, + rotate_lim=[0, 0, 0.2], + ) + self.box = create_box( + scene=self, + pose=rand_pos, + half_size=(0.03, 0.03, 0.1), + color=(1, 0, 0), + name="box", + boxtype="long", + ) + + rand_pos = rand_pose( + xlim=[0.1, 0.25], + ylim=[0.15, 0.2], + ) + + self.target_box = create_box( + scene=self, + pose=rand_pos, + half_size=(0.05, 0.05, 0.005), + color=(0, 0, 1), + name="target_box", + is_static=True, + ) + + self.add_prohibit_area(self.box, padding=0.1) + self.add_prohibit_area(self.target_box, padding=0.1) + self.block_middle_pose = [0, 0.0, 0.9, 0, 1, 0, 0] + + def play_once(self): + # Determine which arm to use for grasping based on box position + grasp_arm_tag = ArmTag("left" if self.box.get_pose().p[0] < 0 else "right") + # The other arm will be used for placing + place_arm_tag = grasp_arm_tag.opposite + + # Grasp the box with the selected arm + self.move( + self.grasp_actor( + self.box, + arm_tag=grasp_arm_tag, + pre_grasp_dis=0.07, + grasp_dis=0.0, + contact_point_id=[0, 1, 2, 3], + )) + # Lift the box up + self.move(self.move_by_displacement(grasp_arm_tag, z=0.1)) + # Place the box at initial position [0, 0., 0.9, 0, 1, 0, 0] + self.move( + self.place_actor( + self.box, + target_pose=self.block_middle_pose, + arm_tag=grasp_arm_tag, + functional_point_id=0, + pre_dis=0, + dis=0, + is_open=False, + constrain="free", + )) + + # Grasp the box again with the other arm (for repositioning) + self.move( + self.grasp_actor( + self.box, + arm_tag=place_arm_tag, + pre_grasp_dis=0.07, + grasp_dis=0.0, + contact_point_id=[4, 5, 6, 7], + )) + # Open the original grasping arm's gripper + self.move(self.open_gripper(grasp_arm_tag)) + # Move the original arm up to release the box + self.move(self.move_by_displacement(grasp_arm_tag, z=0.1, move_axis="arm")) + # Perform two actions simultaneously: + # 1. Return the original arm to its origin position + # 2. Place the box at the target's functional point with precise alignment + self.move( + self.back_to_origin(grasp_arm_tag), + self.place_actor( + self.box, + target_pose=self.target_box.get_functional_point(1, "pose"), + arm_tag=place_arm_tag, + functional_point_id=0, + pre_dis=0.05, + dis=0., + constrain="align", + pre_dis_axis="fp", + ), + ) + + return self.info + + def check_success(self): + box_pos = self.box.get_functional_point(0, "pose").p + target_pose = self.target_box.get_functional_point(1, "pose").p + eps = [0.03, 0.03] + return (np.all(np.abs(box_pos[:2] - target_pose[:2]) < eps) and abs(box_pos[2] - target_pose[2]) < 0.01 + and self.is_right_gripper_open()) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/handover_mic.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/handover_mic.py new file mode 100644 index 00000000..fda34415 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/handover_mic.py @@ -0,0 +1,107 @@ +from ._base_task import Base_Task +from .utils import * +from ._GLOBAL_CONFIGS import * + + +class handover_mic(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + rand_pos = rand_pose( + xlim=[-0.2, 0.2], + ylim=[-0.05, 0.0], + qpos=[0.707, 0.707, 0, 0], + rotate_rand=False, + ) + while abs(rand_pos.p[0]) < 0.15: + rand_pos = rand_pose( + xlim=[-0.2, 0.2], + ylim=[-0.05, 0.0], + qpos=[0.707, 0.707, 0, 0], + rotate_rand=False, + ) + self.microphone_id = np.random.choice([0, 4, 5], 1)[0] + + self.microphone = create_actor( + scene=self, + pose=rand_pos, + modelname="018_microphone", + convex=True, + model_id=self.microphone_id, + ) + + self.add_prohibit_area(self.microphone, padding=0.07) + self.handover_middle_pose = [0, -0.05, 0.98, 0, 1, 0, 0] + self.grasp_arm_tag = ArmTag("right" if self.microphone.get_pose().p[0] > 0 else "left") + self.handover_arm_tag = self.grasp_arm_tag.opposite + + def play_once(self): + # Determine the arm to grasp the microphone based on its position + grasp_arm_tag = ArmTag("right" if self.microphone.get_pose().p[0] > 0 else "left") + # The opposite arm will be used for the handover + handover_arm_tag = grasp_arm_tag.opposite + + # Move the grasping arm to the microphone's position and grasp it + self.move( + self.grasp_actor( + self.microphone, + arm_tag=grasp_arm_tag, + contact_point_id=[1, 9, 10, 11, 12, 13, 14, 15], + pre_grasp_dis=0.1, + )) + # Move the handover arm to a position suitable for handing over the microphone + self.move( + self.move_by_displacement( + grasp_arm_tag, + z=0.12, + quat=(GRASP_DIRECTION_DIC["front_right"] + if grasp_arm_tag == "left" else GRASP_DIRECTION_DIC["front_left"]), + move_axis="arm", + )) + + # Move the handover arm to the middle position for handover + self.move( + self.place_actor( + self.microphone, + arm_tag=grasp_arm_tag, + target_pose=self.handover_middle_pose, + functional_point_id=0, + pre_dis=0.0, + dis=0.0, + is_open=False, + constrain="free", + )) + # Move the handover arm to grasp the microphone from the grasping arm + self.move( + self.grasp_actor( + self.microphone, + arm_tag=handover_arm_tag, + contact_point_id=[0, 2, 3, 4, 5, 6, 7, 8], + pre_grasp_dis=0.1, + )) + # Move the grasping arm to open the gripper and lift the microphone + self.move(self.open_gripper(grasp_arm_tag)) + # Move the handover arm to lift the microphone to a height of 0.98 + self.move( + self.move_by_displacement(grasp_arm_tag, z=0.07, move_axis="arm"), + self.move_by_displacement(handover_arm_tag, x=0.05 if handover_arm_tag == "right" else -0.05), + ) + + self.info["info"] = { + "{A}": f"018_microphone/base{self.microphone_id}", + "{a}": str(grasp_arm_tag), + "{b}": str(handover_arm_tag), + } + return self.info + + def check_success(self): + microphone_pose = self.microphone.get_functional_point(0) + contact = self.get_gripper_actor_contact_position("018_microphone") + if len(contact) == 0: + return False + close_gripper_func = self.is_left_gripper_close if self.handover_arm_tag == "left" else self.is_right_gripper_close + open_gripper_func = self.is_left_gripper_open if self.grasp_arm_tag == "left" else self.is_right_gripper_open + tag = microphone_pose[0] < 0 if self.handover_arm_tag == "left" else microphone_pose[0] > 0 + return (close_gripper_func() and open_gripper_func() and microphone_pose[2] > 0.92 and tag) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/hanging_mug.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/hanging_mug.py new file mode 100644 index 00000000..c553135c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/hanging_mug.py @@ -0,0 +1,88 @@ +from ._base_task import Base_Task +from .utils import * +import numpy as np +from ._GLOBAL_CONFIGS import * + + +class hanging_mug(Base_Task): + + def setup_demo(self, is_test=False, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + self.mug_id = np.random.choice([i for i in range(10)]) + self.mug = rand_create_actor( + self, + xlim=[-0.25, -0.1], + ylim=[-0.05, 0.05], + ylim_prop=True, + modelname="039_mug", + rotate_rand=True, + rotate_lim=[0, 1.57, 0], + qpos=[0.707, 0.707, 0, 0], + convex=True, + model_id=self.mug_id, + ) + + rack_pose = rand_pose( + xlim=[0.1, 0.3], + ylim=[0.13, 0.17], + rotate_rand=True, + rotate_lim=[0, 0.2, 0], + qpos=[-0.22, -0.22, 0.67, 0.67], + ) + + self.rack = create_actor(self, pose=rack_pose, modelname="040_rack", is_static=True, convex=True) + + self.add_prohibit_area(self.mug, padding=0.1) + self.add_prohibit_area(self.rack, padding=0.1) + self.middle_pos = [0.0, -0.15, 0.75, 1, 0, 0, 0] + + def play_once(self): + # Initialize arm tags for grasping and hanging + grasp_arm_tag = ArmTag("left") + hang_arm_tag = ArmTag("right") + + # Move the grasping arm to the mug's position and grasp it + self.move(self.grasp_actor(self.mug, arm_tag=grasp_arm_tag, pre_grasp_dis=0.05)) + self.move(self.move_by_displacement(arm_tag=grasp_arm_tag, z=0.08)) + + # Move the grasping arm to a middle position before hanging + self.move( + self.place_actor(self.mug, + arm_tag=grasp_arm_tag, + target_pose=self.middle_pos, + pre_dis=0.05, + dis=0.0, + constrain="free")) + self.move(self.move_by_displacement(arm_tag=grasp_arm_tag, z=0.1)) + + # Grasp the mug with the hanging arm, and move the grasping arm back to its origin + self.move(self.back_to_origin(grasp_arm_tag), + self.grasp_actor(self.mug, arm_tag=hang_arm_tag, pre_grasp_dis=0.05)) + self.move(self.move_by_displacement(arm_tag=hang_arm_tag, z=0.1, quat=GRASP_DIRECTION_DIC['front'])) + + # Target pose for hanging the mug is the functional point of the rack + target_pose = self.rack.get_functional_point(0) + # Move the hanging arm to the target pose and hang the mug + self.move( + self.place_actor(self.mug, + arm_tag=hang_arm_tag, + target_pose=target_pose, + functional_point_id=0, + constrain="align", + pre_dis=0.05, + dis=-0.05, + pre_dis_axis='fp')) + self.move(self.move_by_displacement(arm_tag=hang_arm_tag, z=0.1, move_axis='arm')) + self.info["info"] = {"{A}": f"039_mug/base{self.mug_id}", "{B}": "040_rack/base0"} + return self.info + + def check_success(self): + mug_function_pose = self.mug.get_functional_point(0)[:3] + rack_pose = self.rack.get_pose().p + rack_function_pose = self.rack.get_functional_point(0)[:3] + rack_middle_pose = (rack_pose + rack_function_pose) / 2 + eps = 0.02 + return (np.all(abs((mug_function_pose - rack_middle_pose)[:2]) < eps) and self.is_right_gripper_open() + and mug_function_pose[2] > 0.86) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/lift_pot.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/lift_pot.py new file mode 100644 index 00000000..30128666 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/lift_pot.py @@ -0,0 +1,58 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math + + +class lift_pot(Base_Task): + + def setup_demo(self, is_test=False, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + self.model_name = "060_kitchenpot" + self.model_id = np.random.randint(0, 2) + self.pot = rand_create_sapien_urdf_obj( + scene=self, + modelname=self.model_name, + modelid=self.model_id, + xlim=[-0.05, 0.05], + ylim=[-0.05, 0.05], + rotate_rand=True, + rotate_lim=[0, 0, np.pi / 8], + qpos=[0.704141, 0, 0, 0.71006], + ) + x, y = self.pot.get_pose().p[0], self.pot.get_pose().p[1] + self.prohibited_area.append([x - 0.3, y - 0.1, x + 0.3, y + 0.1]) + + def play_once(self): + left_arm_tag = ArmTag("left") + right_arm_tag = ArmTag("right") + # Close both left and right grippers to half position + self.move( + self.close_gripper(left_arm_tag, pos=0.5), + self.close_gripper(right_arm_tag, pos=0.5), + ) + # Grasp the pot with both arms at specified contact points + self.move( + self.grasp_actor(self.pot, left_arm_tag, pre_grasp_dis=0.035, contact_point_id=0), + self.grasp_actor(self.pot, right_arm_tag, pre_grasp_dis=0.035, contact_point_id=1), + ) + # Lift the pot by moving both arms upward to target height (0.88) + self.move( + self.move_by_displacement(left_arm_tag, z=0.88 - self.pot.get_pose().p[2]), + self.move_by_displacement(right_arm_tag, z=0.88 - self.pot.get_pose().p[2]), + ) + + self.info["info"] = {"{A}": f"{self.model_name}/base{self.model_id}"} + return self.info + + def check_success(self): + pot_pose = self.pot.get_pose() + left_end = np.array(self.robot.get_left_tcp_pose()[:3]) + right_end = np.array(self.robot.get_right_tcp_pose()[:3]) + left_grasp = np.array(self.pot.get_contact_point(0)[:3]) + right_grasp = np.array(self.pot.get_contact_point(1)[:3]) + pot_dir = get_face_prod(pot_pose.q, [0, 0, 1], [0, 0, 1]) + return (pot_pose.p[2] > 0.82 and np.sqrt(np.sum((left_end - left_grasp)**2)) < 0.03 + and np.sqrt(np.sum((right_end - right_grasp)**2)) < 0.03 and pot_dir > 0.8) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/move_can_pot.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/move_can_pot.py new file mode 100644 index 00000000..896f4de0 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/move_can_pot.py @@ -0,0 +1,110 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math +from copy import deepcopy + + +class move_can_pot(Base_Task): + + def setup_demo(self, is_test=False, **kwargs): + super()._init_task_env_(**kwargs) + + def load_actors(self): + self.pot_id = np.random.randint(0, 7) + self.pot = rand_create_sapien_urdf_obj( + scene=self, + modelname="060_kitchenpot", + modelid=self.pot_id, + xlim=[0.0, 0.0], + ylim=[0.0, 0.0], + rotate_rand=True, + rotate_lim=[0, 0, np.pi / 8], + qpos=[0, 0, 0, 1], + ) + pot_pose = self.pot.get_pose() + rand_pos = rand_pose( + xlim=[-0.3, 0.3], + ylim=[0.05, 0.15], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, np.pi / 4, 0], + ) + while abs(rand_pos.p[0]) < 0.2 or (((pot_pose.p[0] - rand_pos.p[0])**2 + + (pot_pose.p[1] - rand_pos.p[1])**2) < 0.09): + rand_pos = rand_pose( + xlim=[-0.3, 0.3], + ylim=[0.05, 0.15], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, np.pi / 4, 0], + ) + id_list = [0, 2, 4, 5, 6] + self.can_id = np.random.choice(id_list) + self.can = create_actor( + scene=self, + pose=rand_pos, + modelname="105_sauce-can", + convex=True, + model_id=self.can_id, + ) + self.arm_tag = ArmTag("right" if self.can.get_pose().p[0] > 0 else "left") + self.add_prohibit_area(self.pot, padding=0.03) + self.add_prohibit_area(self.can, padding=0.1) + pot_x, pot_y = self.pot.get_pose().p[0], self.pot.get_pose().p[1] + if self.arm_tag == "left": + self.prohibited_area.append([pot_x - 0.15, pot_y - 0.1, pot_x, pot_y + 0.1]) + else: + self.prohibited_area.append([pot_x, pot_y - 0.1, pot_x + 0.15, pot_y + 0.1]) + self.orig_z = self.pot.get_pose().p[2] + + # Get pot's current pose and calculate target pose for placing the can + pot_pose = self.pot.get_pose() + self.target_pose = sapien.Pose( + [ + pot_pose.p[0] - 0.18 if self.arm_tag == "left" else pot_pose.p[0] + 0.18, + pot_pose.p[1], + 0.741 + self.table_z_bias, + ], + pot_pose.q, + ) + + def play_once(self): + arm_tag = self.arm_tag + # Grasp the can with specified pre-grasp distance + self.move(self.grasp_actor(self.can, arm_tag=arm_tag, pre_grasp_dis=0.05)) + # Move the can backward and upward + self.move(self.move_by_displacement(arm_tag, y=-0.1, z=0.1)) + + # Place the can near the pot at calculated target pose + self.move(self.place_actor( + self.can, + target_pose=self.target_pose, + arm_tag=arm_tag, + pre_dis=0.05, + dis=0.0, + )) + + self.info["info"] = { + "{A}": f"060_kitchenpot/base{self.pot_id}", + "{B}": f"105_sauce-can/base{self.can_id}", + "{a}": str(arm_tag), + } + return self.info + + def check_success(self): + pot_pose = self.pot.get_pose().p + can_pose = self.can.get_pose().p + can_pose_rpy = t3d.euler.quat2euler(self.can.get_pose().q) + x_rotate = can_pose_rpy[0] * 180 / np.pi + y_rotate = can_pose_rpy[1] * 180 / np.pi + eps = np.array([0.2, 0.035, 15, 15]) + dis = (pot_pose[0] - can_pose[0] if self.arm_tag == "left" else can_pose[0] - pot_pose[0]) + check = True if dis > 0 else False + return (np.all(np.array([ + abs(dis), + np.abs(pot_pose[1] - can_pose[1]), + abs(x_rotate - 90), + abs(y_rotate), + ]) < eps) and check and can_pose[2] <= self.orig_z + 0.001 and self.robot.is_left_gripper_open() + and self.robot.is_right_gripper_open()) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/move_pillbottle_pad.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/move_pillbottle_pad.py new file mode 100644 index 00000000..7a433f4f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/move_pillbottle_pad.py @@ -0,0 +1,103 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math +from ._GLOBAL_CONFIGS import * +from copy import deepcopy + + +class move_pillbottle_pad(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + rand_pos = rand_pose( + xlim=[-0.25, 0.25], + ylim=[-0.1, 0.1], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=False, + ) + while abs(rand_pos.p[0]) < 0.05: + rand_pos = rand_pose( + xlim=[-0.25, 0.25], + ylim=[-0.1, 0.1], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=False, + ) + + self.pillbottle_id = np.random.choice([1, 2, 3, 4, 5], 1)[0] + self.pillbottle = create_actor( + scene=self, + pose=rand_pos, + modelname="080_pillbottle", + convex=True, + model_id=self.pillbottle_id, + ) + self.pillbottle.set_mass(0.05) + + if rand_pos.p[0] > 0: + xlim = [0.05, 0.25] + else: + xlim = [-0.25, -0.05] + target_rand_pose = rand_pose( + xlim=xlim, + ylim=[-0.2, 0.1], + qpos=[1, 0, 0, 0], + rotate_rand=False, + ) + while (np.sqrt((target_rand_pose.p[0] - rand_pos.p[0])**2 + (target_rand_pose.p[1] - rand_pos.p[1])**2) < 0.1): + target_rand_pose = rand_pose( + xlim=xlim, + ylim=[-0.2, 0.1], + qpos=[1, 0, 0, 0], + rotate_rand=False, + ) + half_size = [0.04, 0.04, 0.0005] + self.pad = create_box( + scene=self, + pose=target_rand_pose, + half_size=half_size, + color=(0, 0, 1), + name="box", + is_static=True, + ) + self.add_prohibit_area(self.pillbottle, padding=0.05) + self.add_prohibit_area(self.pad, padding=0.1) + + def play_once(self): + # Determine which arm to use based on pillbottle's position (right if on right side, left otherwise) + arm_tag = ArmTag("right" if self.pillbottle.get_pose().p[0] > 0 else "left") + + # Grasp the pillbottle + self.move(self.grasp_actor(self.pillbottle, arm_tag=arm_tag, pre_grasp_dis=0.06, gripper_pos=0)) + + # Lift up the pillbottle by 0.1 meters in z-axis + self.move(self.move_by_displacement(arm_tag=arm_tag, z=0.05)) + + # Get the target pose for placing the pillbottle + target_pose = self.pad.get_functional_point(1) + # Place the pillbottle at the target pose + self.move( + self.place_actor(self.pillbottle, + arm_tag=arm_tag, + target_pose=target_pose, + pre_dis=0.05, + dis=0, + functional_point_id=0, + pre_dis_axis='fp')) + + self.info["info"] = { + "{A}": f"080_pillbottle/base{self.pillbottle_id}", + "{a}": str(arm_tag), + } + + return self.info + + def check_success(self): + pillbottle_pos = self.pillbottle.get_pose().p + target_pos = self.pad.get_pose().p + eps1 = 0.03 + return (np.all(abs(pillbottle_pos[:2] - target_pos[:2]) < np.array([eps1, eps1])) + and np.abs(self.pillbottle.get_pose().p[2] - (0.741 + self.table_z_bias)) < 0.005 + and self.robot.is_left_gripper_open() and self.robot.is_right_gripper_open()) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/move_playingcard_away.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/move_playingcard_away.py new file mode 100644 index 00000000..60b59442 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/move_playingcard_away.py @@ -0,0 +1,67 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math +from ._GLOBAL_CONFIGS import * +from copy import deepcopy + + +class move_playingcard_away(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + rand_pos = rand_pose( + xlim=[-0.1, 0.1], + ylim=[-0.2, 0.05], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, 3.14, 0], + ) + while abs(rand_pos.p[0]) < 0.05: + rand_pos = rand_pose( + xlim=[-0.1, 0.1], + ylim=[-0.2, 0.05], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, 3.14, 0], + ) + + self.playingcards_id = np.random.choice([0, 1, 2], 1)[0] + self.playingcards = create_actor( + scene=self, + pose=rand_pos, + modelname="081_playingcards", + convex=True, + model_id=self.playingcards_id, + ) + + self.prohibited_area.append([-100, -0.3, 100, 0.1]) + self.add_prohibit_area(self.playingcards, padding=0.1) + + self.target_pose = self.playingcards.get_pose() # TODO + + def play_once(self): + # Determine which arm to use based on playing cards position + arm_tag = ArmTag("right" if self.playingcards.get_pose().p[0] > 0 else "left") + + # Grasp the playing cards with specified arm + self.move(self.grasp_actor(self.playingcards, arm_tag=arm_tag, pre_grasp_dis=0.1, grasp_dis=0.01)) + # Move the playing cards horizontally (right if right arm, left if left arm) + self.move(self.move_by_displacement(arm_tag, x=0.3 if arm_tag == "right" else -0.3)) + # Open gripper to release the playing cards + self.move(self.open_gripper(arm_tag)) + + self.info["info"] = { + "{A}": f"081_playingcards/base{self.playingcards_id}", + "{a}": str(arm_tag), + } + return self.info + + def check_success(self): + playingcards_pose = self.playingcards.get_pose().p + edge_x = 0.23 + + return (np.all(abs(playingcards_pose[0]) > abs(edge_x)) and self.robot.is_left_gripper_open() + and self.robot.is_right_gripper_open()) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/move_stapler_pad.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/move_stapler_pad.py new file mode 100644 index 00000000..02401630 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/move_stapler_pad.py @@ -0,0 +1,120 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math +from ._GLOBAL_CONFIGS import * +from copy import deepcopy + + +class move_stapler_pad(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + rand_pos = rand_pose( + xlim=[-0.25, 0.25], + ylim=[-0.2, 0.0], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, 3.14, 0], + ) + while abs(rand_pos.p[0]) < 0.05: + rand_pos = rand_pose( + xlim=[-0.25, 0.25], + ylim=[-0.2, 0.0], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, 3.14, 0], + ) + self.stapler_id = np.random.choice([0, 1, 2, 3, 4, 5, 6], 1)[0] + self.stapler = create_actor( + scene=self, + pose=rand_pos, + modelname="048_stapler", + convex=True, + model_id=self.stapler_id, + ) + + if rand_pos.p[0] > 0: + xlim = [0.05, 0.25] + else: + xlim = [-0.25, -0.05] + target_rand_pose = rand_pose( + xlim=xlim, + ylim=[-0.2, 0.0], + qpos=[1, 0, 0, 0], + rotate_rand=False, + ) + while (np.sqrt((target_rand_pose.p[0] - rand_pos.p[0])**2 + (target_rand_pose.p[1] - rand_pos.p[1])**2) < 0.1): + target_rand_pose = rand_pose( + xlim=xlim, + ylim=[-0.2, 0.0], + qpos=[1, 0, 0, 0], + rotate_rand=False, + ) + half_size = [0.055, 0.03, 0.0005] + + colors = { + "Red": (1, 0, 0), + "Green": (0, 1, 0), + "Blue": (0, 0, 1), + "Yellow": (1, 1, 0), + "Cyan": (0, 1, 1), + "Magenta": (1, 0, 1), + "Black": (0, 0, 0), + "Gray": (0.5, 0.5, 0.5), + } + + color_items = list(colors.items()) + color_index = np.random.choice(len(color_items)) + self.color_name, self.color_value = color_items[color_index] + + self.pad = create_box( + scene=self.scene, + pose=target_rand_pose, + half_size=half_size, + color=self.color_value, + name="box", + ) + self.add_prohibit_area(self.stapler, padding=0.1) + self.add_prohibit_area(self.pad, padding=0.15) + + # Create target pose by combining target position with default quaternion orientation + self.pad_pose = self.pad.get_pose().p.tolist() + [0.707, 0, 0, 0.707] + + def play_once(self): + # Determine which arm to use based on stapler's position (right if on positive x, left otherwise) + arm_tag = ArmTag("right" if self.stapler.get_pose().p[0] > 0 else "left") + + # Grasp the stapler with specified arm + self.move(self.grasp_actor(self.stapler, arm_tag=arm_tag, pre_grasp_dis=0.1)) + # Move the arm upward by 0.1 meters along z-axis + self.move(self.move_by_displacement(arm_tag, z=0.1, move_axis="arm")) + + # Place the stapler at target pose with alignment constraint + self.move( + self.place_actor( + self.stapler, + target_pose=self.pad_pose, + arm_tag=arm_tag, + pre_dis=0.1, + dis=0.0, + constrain="align", + )) + + self.info["info"] = { + "{A}": f"048_stapler/base{self.stapler_id}", + "{B}": self.color_name, + "{a}": str(arm_tag), + } + return self.info + + def check_success(self): + stapler_pose = self.stapler.get_pose().p + stapler_qpose = np.abs(self.stapler.get_pose().q) + target_pos = self.pad.get_pose().p + eps = [0.02, 0.02, 0.01] + return (np.all(abs(stapler_pose - target_pos) < np.array(eps)) + and (stapler_qpose.max() - stapler_qpose.min()) < 0.02 and self.robot.is_left_gripper_open() + and self.robot.is_right_gripper_open()) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/open_laptop.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/open_laptop.py new file mode 100644 index 00000000..3a1f1876 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/open_laptop.py @@ -0,0 +1,67 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math + + +class open_laptop(Base_Task): + + def setup_demo(self, is_test=False, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + self.model_name = "015_laptop" + self.model_id = np.random.randint(0, 11) + self.laptop: ArticulationActor = rand_create_sapien_urdf_obj( + scene=self, + modelname=self.model_name, + modelid=self.model_id, + xlim=[-0.05, 0.05], + ylim=[-0.1, 0.05], + rotate_rand=True, + rotate_lim=[0, 0, np.pi / 3], + qpos=[0.7, 0, 0, 0.7], + fix_root_link=True, + ) + limit = self.laptop.get_qlimits()[0] + self.laptop.set_qpos([limit[0] + (limit[1] - limit[0]) * 0.2]) + self.laptop.set_mass(0.01) + self.laptop.set_properties(1, 0) + self.add_prohibit_area(self.laptop, padding=0.1) + + def play_once(self): + face_prod = get_face_prod(self.laptop.get_pose().q, [1, 0, 0], [1, 0, 0]) + arm_tag = ArmTag("left" if face_prod > 0 else "right") + self.arm_tag = arm_tag + + # Grasp the laptop + self.move(self.grasp_actor(self.laptop, arm_tag=arm_tag, pre_grasp_dis=0.08, contact_point_id=0)) + + for _ in range(15): + # Get target rotation pose + self.move( + self.grasp_actor( + self.laptop, + arm_tag=arm_tag, + pre_grasp_dis=0.0, + grasp_dis=0.0, + contact_point_id=1, + )) + if not self.plan_success: + break + if self.check_success(target=0.5): + break + + self.info["info"] = { + "{A}": f"{self.model_name}/base{self.model_id}", + "{a}": str(arm_tag), + } + return self.info + + def check_success(self, target=0.4): + limit = self.laptop.get_qlimits()[0] + qpos = self.laptop.get_qpos() + rotate_pose = self.laptop.get_contact_point(1) + tip_pose = (self.robot.get_left_tcp_pose() if self.arm_tag == "left" else self.robot.get_right_tcp_pose()) + dis = np.sqrt(np.sum((np.array(tip_pose[:3]) - np.array(rotate_pose[:3]))**2)) + return qpos[0] >= limit[0] + (limit[1] - limit[0]) * target and dis < 0.1 diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/open_microwave.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/open_microwave.py new file mode 100644 index 00000000..6cdace39 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/open_microwave.py @@ -0,0 +1,105 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math + + +class open_microwave(Base_Task): + + def setup_demo(self, is_test=False, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + self.model_name = "044_microwave" + self.model_id = np.random.randint(0, 2) + self.microwave = rand_create_sapien_urdf_obj( + scene=self, + modelname=self.model_name, + modelid=self.model_id, + xlim=[-0.12, -0.02], + ylim=[0.15, 0.2], + zlim=[0.8, 0.8], + qpos=[0.707, 0, 0, 0.707], + fix_root_link=True, + ) + self.microwave.set_mass(0.01) + self.microwave.set_properties(0.0, 0.0) + + self.add_prohibit_area(self.microwave) + self.prohibited_area.append([-0.25, -0.25, 0.25, 0.1]) + + def play_once(self): + arm_tag = ArmTag("left") + + # Grasp the microwave with pre-grasp displacement + self.move(self.grasp_actor(self.microwave, arm_tag=arm_tag, pre_grasp_dis=0.08, contact_point_id=0)) + + start_qpos = self.microwave.get_qpos()[0] + for _ in range(50): + # Rotate microwave + self.move( + self.grasp_actor( + self.microwave, + arm_tag=arm_tag, + pre_grasp_dis=0.0, + grasp_dis=0.0, + contact_point_id=4, + )) + + new_qpos = self.microwave.get_qpos()[0] + if new_qpos - start_qpos <= 0.001: + break + start_qpos = new_qpos + if not self.plan_success: + break + if self.check_success(target=0.7): + break + + if not self.check_success(target=0.7): + self.plan_success = True # Try new way + # Open gripper + self.move(self.open_gripper(arm_tag=arm_tag)) + self.move(self.move_by_displacement(arm_tag=arm_tag, y=-0.05, z=0.05)) + + # Grasp at contact point 1 + self.move(self.grasp_actor(self.microwave, arm_tag=arm_tag, contact_point_id=1)) + + # Grasp more tightly at contact point 1 + self.move(self.grasp_actor( + self.microwave, + arm_tag=arm_tag, + pre_grasp_dis=0.02, + contact_point_id=1, + )) + + start_qpos = self.microwave.get_qpos()[0] + for _ in range(30): + # Rotate microwave using contact point 2 + self.move( + self.grasp_actor( + self.microwave, + arm_tag=arm_tag, + pre_grasp_dis=0.0, + grasp_dis=0.0, + contact_point_id=2, + )) + + new_qpos = self.microwave.get_qpos()[0] + if new_qpos - start_qpos <= 0.001: + break + start_qpos = new_qpos + if not self.plan_success: + break + if self.check_success(target=0.7): + break + + self.info["info"] = { + "{A}": f"{self.model_name}/base{self.model_id}", + "{a}": str(arm_tag), + } + return self.info + + def check_success(self, target=0.6): + limits = self.microwave.get_qlimits() + qpos = self.microwave.get_qpos() + return qpos[0] >= limits[0][1] * target diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/pick_diverse_bottles.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/pick_diverse_bottles.py new file mode 100644 index 00000000..0b90707d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/pick_diverse_bottles.py @@ -0,0 +1,104 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +from copy import deepcopy + + +class pick_diverse_bottles(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + self.id_list = [i for i in range(20)] + self.bottle1_id = np.random.choice(self.id_list) + self.bottle2_id = np.random.choice(self.id_list) + self.bottle1 = rand_create_actor( + self, + xlim=[-0.25, -0.05], + ylim=[0.03, 0.23], + modelname="001_bottle", + rotate_rand=True, + rotate_lim=[0, 1, 0], + qpos=[0.66, 0.66, -0.25, -0.25], + convex=True, + model_id=self.bottle1_id, + ) + + self.bottle2 = rand_create_actor( + self, + xlim=[0.05, 0.25], + ylim=[0.03, 0.23], + modelname="001_bottle", + rotate_rand=True, + rotate_lim=[0, 1, 0], + qpos=[0.65, 0.65, 0.27, 0.27], + convex=True, + model_id=self.bottle2_id, + ) + + self.delay(4) + + self.add_prohibit_area(self.bottle1, padding=0.1) + self.add_prohibit_area(self.bottle2, padding=0.1) + target_posi = [-0.2, -0.2, 0.2, -0.02] + self.prohibited_area.append(target_posi) + self.left_target_pose = [-0.06, -0.105, 1, 0, 1, 0, 0] + self.right_target_pose = [0.06, -0.105, 1, 0, 1, 0, 0] + + def play_once(self): + # Determine which arm to use for each bottle based on their x-coordinate position + bottle1_arm_tag = ArmTag("left") + bottle2_arm_tag = ArmTag("right") + + # Grasp both bottles simultaneously with their respective arms + self.move( + self.grasp_actor(self.bottle1, arm_tag=bottle1_arm_tag, pre_grasp_dis=0.08), + self.grasp_actor(self.bottle2, arm_tag=bottle2_arm_tag, pre_grasp_dis=0.08), + ) + + # Lift both bottles up simultaneously + self.move( + self.move_by_displacement(arm_tag=bottle1_arm_tag, z=0.1), + self.move_by_displacement(arm_tag=bottle2_arm_tag, z=0.1), + ) + + # Place both bottles to their target positions simultaneously + self.move( + self.place_actor( + self.bottle1, + target_pose=self.left_target_pose, + arm_tag=bottle1_arm_tag, + functional_point_id=0, + pre_dis=0.0, + dis=0.0, + is_open=False, + ), + self.place_actor( + self.bottle2, + target_pose=self.right_target_pose, + arm_tag=bottle2_arm_tag, + functional_point_id=0, + pre_dis=0.0, + dis=0.0, + is_open=False, + ), + ) + + self.info["info"] = { + "{A}": f"001_bottle/base{self.bottle1_id}", + "{B}": f"001_bottle/base{self.bottle2_id}", + } + return self.info + + def check_success(self): + bottle1_target = self.left_target_pose[:2] + bottle2_target = self.right_target_pose[:2] + eps = 0.1 + bottle1_pose = self.bottle1.get_functional_point(0) + bottle2_pose = self.bottle2.get_functional_point(0) + if bottle1_pose[2] < 0.78 or bottle2_pose[2] < 0.78: + self.actor_pose = False + return (abs(bottle1_pose[0] - bottle1_target[0]) < eps and abs(bottle1_pose[1] - bottle1_target[1]) < eps + and bottle1_pose[2] > 0.89 and abs(bottle2_pose[0] - bottle2_target[0]) < eps + and abs(bottle2_pose[1] - bottle2_target[1]) < eps and bottle2_pose[2] > 0.89) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/pick_dual_bottles.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/pick_dual_bottles.py new file mode 100644 index 00000000..f1b7428b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/pick_dual_bottles.py @@ -0,0 +1,102 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +from copy import deepcopy + + +class pick_dual_bottles(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + self.bottle1 = rand_create_actor( + self, + xlim=[-0.25, -0.05], + ylim=[0.03, 0.23], + modelname="001_bottle", + rotate_rand=True, + rotate_lim=[0, 1, 0], + qpos=[0.66, 0.66, -0.25, -0.25], + convex=True, + model_id=13, + ) + + self.bottle2 = rand_create_actor( + self, + xlim=[0.05, 0.25], + ylim=[0.03, 0.23], + modelname="001_bottle", + rotate_rand=True, + rotate_lim=[0, 1, 0], + qpos=[0.65, 0.65, 0.27, 0.27], + convex=True, + model_id=16, + ) + + render_freq = self.render_freq + self.render_freq = 0 + for _ in range(4): + self.together_open_gripper(save_freq=None) + self.render_freq = render_freq + + self.add_prohibit_area(self.bottle1, padding=0.1) + self.add_prohibit_area(self.bottle2, padding=0.1) + target_posi = [-0.2, -0.2, 0.2, -0.02] + self.prohibited_area.append(target_posi) + self.left_target_pose = [-0.06, -0.105, 1, 0, 1, 0, 0] + self.right_target_pose = [0.06, -0.105, 1, 0, 1, 0, 0] + + def play_once(self): + # Determine which arm to use for each bottle based on their x-coordinate position + bottle1_arm_tag = ArmTag("left") + bottle2_arm_tag = ArmTag("right") + + # Simultaneously grasp both bottles with their respective arms + self.move( + self.grasp_actor(self.bottle1, arm_tag=bottle1_arm_tag, pre_grasp_dis=0.08), + self.grasp_actor(self.bottle2, arm_tag=bottle2_arm_tag, pre_grasp_dis=0.08), + ) + + # Simultaneously lift both bottles up by 0.1 meters + self.move( + self.move_by_displacement(arm_tag=bottle1_arm_tag, z=0.1), + self.move_by_displacement(arm_tag=bottle2_arm_tag, z=0.1), + ) + + # Simultaneously place both bottles at their target positions + self.move( + self.place_actor( + self.bottle1, + target_pose=self.left_target_pose, + arm_tag=bottle1_arm_tag, + functional_point_id=0, + pre_dis=0.0, + dis=0.0, + is_open=False, + ), + self.place_actor( + self.bottle2, + target_pose=self.right_target_pose, + arm_tag=bottle2_arm_tag, + functional_point_id=0, + pre_dis=0.0, + dis=0.0, + is_open=False, + ), + ) + + self.info["info"] = {"{A}": f"001_bottle/base13", "{B}": f"001_bottle/base16"} + return self.info + + def check_success(self): + bottle1_target = self.left_target_pose[:2] + bottle2_target = self.right_target_pose[:2] + eps = 0.1 + bottle1_pose = self.bottle1.get_functional_point(0) + bottle2_pose = self.bottle2.get_functional_point(0) + if bottle1_pose[2] < 0.78 or bottle2_pose[2] < 0.78: + self.actor_pose = False + return (abs(bottle1_pose[0] - bottle1_target[0]) < eps and abs(bottle1_pose[1] - bottle1_target[1]) < eps + and bottle1_pose[2] > 0.89 and abs(bottle2_pose[0] - bottle2_target[0]) < eps + and abs(bottle2_pose[1] - bottle2_target[1]) < eps and bottle2_pose[2] > 0.89) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_a2b_left.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_a2b_left.py new file mode 100644 index 00000000..d60e7885 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_a2b_left.py @@ -0,0 +1,153 @@ +import glob +from ._base_task import Base_Task +from .utils import * +import sapien +import math +from ._GLOBAL_CONFIGS import * +from copy import deepcopy +import numpy as np + + +class place_a2b_left(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + + def get_available_model_ids(modelname): + asset_path = os.path.join("assets/objects", modelname) + json_files = glob.glob(os.path.join(asset_path, "model_data*.json")) + + available_ids = [] + for file in json_files: + base = os.path.basename(file) + try: + idx = int(base.replace("model_data", "").replace(".json", "")) + available_ids.append(idx) + except ValueError: + continue + return available_ids + + object_list = [ + "047_mouse", + "048_stapler", + "050_bell", + "057_toycar", + "073_rubikscube", + "075_bread", + "077_phone", + "081_playingcards", + "086_woodenblock", + "112_tea-box", + "113_coffee-box", + "107_soap", + ] + + try_num, try_lim = 0, 100 + while try_num <= try_lim: + rand_pos = rand_pose( + xlim=[-0.22, 0.22], + ylim=[-0.2, 0.0], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, 3.14, 0], + ) + if rand_pos.p[0] > 0: + xlim = [0.18, 0.23] + else: + xlim = [-0.1, 0.1] + target_rand_pose = rand_pose( + xlim=xlim, + ylim=[-0.2, 0.0], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, 3.14, 0], + ) + while (np.sqrt((target_rand_pose.p[0] - rand_pos.p[0])**2 + (target_rand_pose.p[1] - rand_pos.p[1])**2) + < 0.1) or (np.abs(target_rand_pose.p[1] - rand_pos.p[1]) < 0.1): + target_rand_pose = rand_pose( + xlim=xlim, + ylim=[-0.2, 0.0], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, 3.14, 0], + ) + try_num += 1 + + distance = np.sqrt(np.sum((rand_pos.p[:2] - target_rand_pose.p[:2])**2)) + + if distance > 0.19 or rand_pos.p[0] > target_rand_pose.p[0]: + break + + if try_num > try_lim: + raise "Actor create limit!" + + self.selected_modelname_A = np.random.choice(object_list) + + available_model_ids = get_available_model_ids(self.selected_modelname_A) + if not available_model_ids: + raise ValueError(f"No available model_data.json files found for {self.selected_modelname_A}") + + self.selected_model_id_A = np.random.choice(available_model_ids) + self.object = create_actor( + scene=self, + pose=rand_pos, + modelname=self.selected_modelname_A, + convex=True, + model_id=self.selected_model_id_A, + ) + + self.selected_modelname_B = np.random.choice(object_list) + while self.selected_modelname_B == self.selected_modelname_A: + self.selected_modelname_B = np.random.choice(object_list) + + available_model_ids = get_available_model_ids(self.selected_modelname_B) + if not available_model_ids: + raise ValueError(f"No available model_data.json files found for {self.selected_modelname_B}") + + self.selected_model_id_B = np.random.choice(available_model_ids) + + self.target_object = create_actor( + scene=self, + pose=target_rand_pose, + modelname=self.selected_modelname_B, + convex=True, + model_id=self.selected_model_id_B, + ) + self.object.set_mass(0.05) + self.target_object.set_mass(0.05) + self.add_prohibit_area(self.object, padding=0.05) + self.add_prohibit_area(self.target_object, padding=0.1) + + def play_once(self): + # Determine which arm to use based on object's x position + arm_tag = ArmTag("right" if self.object.get_pose().p[0] > 0 else "left") + + # Grasp the object with specified arm + self.move(self.grasp_actor(self.object, arm_tag=arm_tag, pre_grasp_dis=0.1)) + # Lift the object upward by 0.1 meters along z-axis using arm movement + self.move(self.move_by_displacement(arm_tag=arm_tag, z=0.1, move_axis="arm")) + + # Get target pose and adjust x position to place object to the left of target + target_pose = self.target_object.get_pose().p.tolist() + target_pose[0] -= 0.13 + + # Place the object at the adjusted target position + self.move(self.place_actor(self.object, arm_tag=arm_tag, target_pose=target_pose)) + + # Record task information including object IDs and used arm + self.info["info"] = { + "{A}": f"{self.selected_modelname_A}/base{self.selected_model_id_A}", + "{B}": f"{self.selected_modelname_B}/base{self.selected_model_id_B}", + "{a}": str(arm_tag), + } + return self.info + + def check_success(self): + object_pose = self.object.get_pose().p + target_pos = self.target_object.get_pose().p + distance = np.sqrt(np.sum((object_pose[:2] - target_pos[:2])**2)) + return np.all(distance < 0.2 and distance > 0.08 and object_pose[0] < target_pos[0] + and abs(object_pose[1] - target_pos[1]) < 0.05 and self.robot.is_left_gripper_open() + and self.robot.is_right_gripper_open()) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_a2b_right.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_a2b_right.py new file mode 100644 index 00000000..8af690ef --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_a2b_right.py @@ -0,0 +1,154 @@ +import glob +from ._base_task import Base_Task +from .utils import * +import sapien +import math +from ._GLOBAL_CONFIGS import * +from copy import deepcopy +import numpy as np + + +class place_a2b_right(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + + def get_available_model_ids(modelname): + asset_path = os.path.join("assets/objects", modelname) + json_files = glob.glob(os.path.join(asset_path, "model_data*.json")) + + available_ids = [] + for file in json_files: + base = os.path.basename(file) + try: + idx = int(base.replace("model_data", "").replace(".json", "")) + available_ids.append(idx) + except ValueError: + continue + return available_ids + + object_list = [ + "047_mouse", + "048_stapler", + "050_bell", + "057_toycar", + "073_rubikscube", + "075_bread", + "077_phone", + "081_playingcards", + "086_woodenblock", + "112_tea-box", + "113_coffee-box", + "107_soap", + ] + object_list_np = np.array(object_list) + + try_num, try_lim = 0, 100 + while try_num <= try_lim: + rand_pos = rand_pose( + xlim=[-0.22, 0.22], + ylim=[-0.2, 0.0], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, 3.14, 0], + ) + if rand_pos.p[0] > 0: + xlim = [-0.1, 0.1] + else: + xlim = [-0.23, -0.18] + target_rand_pose = rand_pose( + xlim=xlim, + ylim=[-0.2, 0.0], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, 3.14, 0], + ) + while (np.sqrt((target_rand_pose.p[0] - rand_pos.p[0])**2 + (target_rand_pose.p[1] - rand_pos.p[1])**2) + < 0.1) or (np.abs(target_rand_pose.p[1] - rand_pos.p[1]) < 0.1): + target_rand_pose = rand_pose( + xlim=xlim, + ylim=[-0.2, 0.0], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, 3.14, 0], + ) + try_num += 1 + + distance = np.sqrt(np.sum((rand_pos.p[:2] - target_rand_pose.p[:2])**2)) + + if distance > 0.19 or rand_pos.p[0] < target_rand_pose.p[0]: + break + + if try_num > try_lim: + raise "Actor create limit!" + + self.selected_modelname_A = np.random.choice(object_list_np) + available_model_ids = get_available_model_ids(self.selected_modelname_A) + self.selected_model_id_A = np.random.choice(available_model_ids) + if not available_model_ids: + raise ValueError(f"No available model_data.json files found for {self.selected_modelname_A}") + + self.object = create_actor( + scene=self, + pose=rand_pos, + modelname=self.selected_modelname_A, + convex=True, + model_id=self.selected_model_id_A, + ) + + self.selected_modelname_B = np.random.choice(object_list_np) + while self.selected_modelname_B == self.selected_modelname_A: + self.selected_modelname_B = np.random.choice(object_list_np) + + available_model_ids = get_available_model_ids(self.selected_modelname_B) + if not available_model_ids: + raise ValueError(f"No available model_data.json files found for {self.selected_modelname_B}") + + self.selected_model_id_B = np.random.choice(available_model_ids) + + self.target_object = create_actor( + scene=self, + pose=target_rand_pose, + modelname=self.selected_modelname_B, + convex=True, + model_id=self.selected_model_id_B, + ) + + self.object.set_mass(0.05) + self.target_object.set_mass(0.05) + self.add_prohibit_area(self.object, padding=0.05) + self.add_prohibit_area(self.target_object, padding=0.1) + + def play_once(self): + # Determine which arm to use based on object's x position (right if positive, left if negative) + arm_tag = ArmTag("right" if self.object.get_pose().p[0] > 0 else "left") + + # Grasp the object with specified arm using pre-grasp distance of 0.1 + self.move(self.grasp_actor(self.object, arm_tag=arm_tag, pre_grasp_dis=0.1)) + # Lift the object upward by 0.1 units along z-axis using arm movement + self.move(self.move_by_displacement(arm_tag=arm_tag, z=0.1, move_axis="arm")) + + # Calculate the target place pose by offsetting target's x position by +0.13 + target_pose = self.target_object.get_pose().p.tolist() + target_pose[0] += 0.13 + + # Place the object at the calculated target pose + self.move(self.place_actor(self.object, arm_tag=arm_tag, target_pose=target_pose)) + + # Store information about the objects and arm used in the info dictionary + self.info["info"] = { + "{A}": f"{self.selected_modelname_A}/base{self.selected_model_id_A}", + "{B}": f"{self.selected_modelname_B}/base{self.selected_model_id_B}", + "{a}": str(arm_tag), + } + return self.info + + def check_success(self): + object_pose = self.object.get_pose().p + target_pos = self.target_object.get_pose().p + distance = np.sqrt(np.sum((object_pose[:2] - target_pos[:2])**2)) + return np.all(distance < 0.2 and distance > 0.08 and object_pose[0] > target_pos[0] + and abs(object_pose[1] - target_pos[1]) < 0.05 and self.robot.is_left_gripper_open() + and self.robot.is_right_gripper_open()) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_bread_basket.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_bread_basket.py new file mode 100644 index 00000000..cb650b49 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_bread_basket.py @@ -0,0 +1,202 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math +from copy import deepcopy +import numpy as np + + +class place_bread_basket(Base_Task): + + def setup_demo(self, **kwargs): + super()._init_task_env_(**kwargs) + + def load_actors(self): + rand_pos = rand_pose( + xlim=[0.0, 0.0], + ylim=[-0.2, -0.2], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, 3.14, 0], + ) + id_list = [0, 1, 2, 3, 4] + self.basket_id = np.random.choice(id_list) + self.breadbasket = create_actor( + scene=self, + pose=rand_pos, + modelname="076_breadbasket", + convex=True, + model_id=self.basket_id, + ) + + breadbasket_pose = self.breadbasket.get_pose() + self.bread: list[Actor] = [] + self.bread_id = [] + + for i in range(2): + rand_pos = rand_pose( + xlim=[-0.27, 0.27], + ylim=[-0.2, 0.05], + qpos=[0.707, 0.707, 0.0, 0.0], + rotate_rand=True, + rotate_lim=[0, np.pi / 4, 0], + ) + try_num = 0 + while True: + pd = True + try_num += 1 + if try_num > 50: + try_num = -1 + break + try_num0 = 0 + while (abs(rand_pos.p[0]) < 0.15 or ((rand_pos.p[0] - breadbasket_pose.p[0])**2 + + (rand_pos.p[1] - breadbasket_pose.p[1])**2) < 0.01): + try_num0 += 1 + rand_pos = rand_pose( + xlim=[-0.27, 0.27], + ylim=[-0.2, 0.05], + qpos=[0.707, 0.707, 0.0, 0.0], + rotate_rand=True, + rotate_lim=[0, np.pi / 4, 0], + ) + if try_num0 > 50: + try_num = -1 + break + if try_num == -1: + break + for j in range(len(self.bread)): + peer_pose = self.bread[j].get_pose() + if ((peer_pose.p[0] - rand_pos.p[0])**2 + (peer_pose.p[1] - rand_pos.p[1])**2) < 0.01: + pd = False + break + if pd: + break + if try_num == -1: + break + id_list = [0, 1, 3, 5, 6] + self.bread_id.append(np.random.choice(id_list)) + bread_actor = create_actor( + scene=self, + pose=rand_pos, + modelname="075_bread", + convex=True, + model_id=self.bread_id[i], + ) + self.bread.append(bread_actor) + + for i in range(len(self.bread)): + self.add_prohibit_area(self.bread[i], padding=0.03) + + self.add_prohibit_area(self.breadbasket, padding=0.05) + + def play_once(self): + + def remove_bread(id, num): + arm_tag = ArmTag("right" if self.bread[id].get_pose().p[0] > 0 else "left") + + # Grasp the bread + self.move(self.grasp_actor(self.bread[id], arm_tag=arm_tag, pre_grasp_dis=0.07)) + # Move up a little + self.move(self.move_by_displacement(arm_tag=arm_tag, z=0.1, move_axis="arm")) + + # Get bread basket's functional point as target pose + breadbasket_pose = self.breadbasket.get_functional_point(0) + # Place the bread into the bread basket + self.move( + self.place_actor( + self.bread[id], + arm_tag=arm_tag, + target_pose=breadbasket_pose, + constrain="free", + pre_dis=0.12, + )) + if num == 0: + # Move up further after placing first bread + self.move(self.move_by_displacement(arm_tag=arm_tag, z=0.15, move_axis="arm")) + else: + # Open gripper to place the second bread + self.move(self.open_gripper(arm_tag=arm_tag)) + + def remove(): + # Determine which bread is on the left + id = 0 if self.bread[0].get_pose().p[0] < 0 else 1 + + # Simultaneously grasp both breads with dual arms + self.move( + self.grasp_actor(self.bread[id], arm_tag="left", pre_grasp_dis=0.05), + self.grasp_actor(self.bread[id ^ 1], arm_tag="right", pre_grasp_dis=0.07), + ) + + # Lift both arms slightly after grasping + self.move( + self.move_by_displacement(arm_tag="left", z=0.05, move_axis="arm"), + self.move_by_displacement(arm_tag="right", z=0.05, move_axis="arm"), + ) + + breadbasket_pose = self.breadbasket.get_functional_point(0) + # Place first bread into the basket using left arm + self.move( + self.place_actor( + self.bread[id], + arm_tag="left", + target_pose=breadbasket_pose, + constrain="free", + pre_dis=0.13, + )) + # Move left arm up a little + self.move(self.move_by_displacement(arm_tag="left", z=0.1, move_axis="arm")) + + # Move left arm away while placing second bread with right arm, avoiding collision + self.move( + self.back_to_origin(arm_tag="left"), + self.place_actor( + self.bread[id ^ 1], + arm_tag="right", + target_pose=breadbasket_pose, + constrain="free", + pre_dis=0.13, + dis=0.05, # Move right arm slightly away to avoid collision + ), + ) + + arm_info = None + # Check if there's only one bread or both are on the same side + if (len(self.bread) <= 1 or (self.bread[0].get_pose().p[0] * self.bread[1].get_pose().p[0]) > 0): + if len(self.bread) == 1: + # Handle single bread case + remove_bread(0, 0) + arm_info = "left" if self.bread[0].get_pose().p[0] < 0 else "right" + else: + # When two breads are present but on the same side, pick the front one first + id = (0 if self.bread[0].get_pose().p[1] < self.bread[1].get_pose().p[1] else 1) + arm_info = "left" if self.bread[0].get_pose().p[0] < 0 else "right" + remove_bread(id, 0) + remove_bread(id ^ 1, 1) + else: + # Dual-arm removal when breads are on opposite sides + remove() + arm_info = "dual" + + self.info["info"] = { + "{A}": f"076_breadbasket/base{self.basket_id}", + "{B}": f"075_bread/base{self.bread_id[0]}", + "{a}": arm_info, + } + if len(self.bread) == 2: + self.info["info"]["{C}"] = f"075_bread/base{self.bread_id[1]}" + + return self.info + + def check_success(self): + breadbasket_pose = self.breadbasket.get_pose().p + eps1 = 0.05 + check = True + for i in range(len(self.bread)): + pose = self.bread[i].get_pose().p + if np.all(abs(pose[:2] - breadbasket_pose[:2]) < np.array([eps1, eps1])) and (pose[2] + > 0.73 + self.table_z_bias): + continue + else: + check = False + + return (check and self.robot.is_left_gripper_open() and self.robot.is_right_gripper_open()) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_bread_skillet.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_bread_skillet.py new file mode 100644 index 00000000..031964c1 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_bread_skillet.py @@ -0,0 +1,118 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import glob + + +class place_bread_skillet(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags, table_static=False) + + def load_actors(self): + id_list = [0, 1, 3, 5, 6] + self.bread_id = np.random.choice(id_list) + rand_pos = rand_pose( + xlim=[-0.28, 0.28], + ylim=[-0.2, 0.05], + qpos=[0.707, 0.707, 0.0, 0.0], + rotate_rand=True, + rotate_lim=[0, np.pi / 4, 0], + ) + while abs(rand_pos.p[0]) < 0.2: + rand_pos = rand_pose( + xlim=[-0.28, 0.28], + ylim=[-0.2, 0.05], + qpos=[0.707, 0.707, 0.0, 0.0], + rotate_rand=True, + rotate_lim=[0, np.pi / 4, 0], + ) + self.bread = create_actor( + self, + pose=rand_pos, + modelname="075_bread", + model_id=self.bread_id, + convex=True, + ) + + xlim = [0.15, 0.25] if rand_pos.p[0] < 0 else [-0.25, -0.15] + self.model_id_list = [0, 1, 2, 3] + self.skillet_id = np.random.choice(self.model_id_list) + rand_pos = rand_pose( + xlim=xlim, + ylim=[-0.2, 0.05], + qpos=[0, 0, 0.707, 0.707], + rotate_rand=True, + rotate_lim=[0, np.pi / 6, 0], + ) + self.skillet = create_actor( + self, + pose=rand_pos, + modelname="106_skillet", + model_id=self.skillet_id, + convex=True, + ) + + self.bread.set_mass(0.001) + self.skillet.set_mass(0.01) + self.add_prohibit_area(self.bread, padding=0.03) + self.add_prohibit_area(self.skillet, padding=0.05) + + def play_once(self): + # Determine which arm to use based on skillet position (right if on positive x, left otherwise) + arm_tag = ArmTag("right" if self.skillet.get_pose().p[0] > 0 else "left") + + # Grasp the skillet and bread simultaneously with dual arms + self.move( + self.grasp_actor(self.skillet, arm_tag=arm_tag, pre_grasp_dis=0.07, gripper_pos=0), + self.grasp_actor(self.bread, arm_tag=arm_tag.opposite, pre_grasp_dis=0.07, gripper_pos=0), + ) + + # Lift both arms + self.move( + self.move_by_displacement(arm_tag=arm_tag, z=0.1, move_axis="arm"), + self.move_by_displacement(arm_tag=arm_tag.opposite, z=0.1), + ) + + # Define a custom target pose for the skillet based on its side (left or right) + target_pose = self.get_arm_pose(arm_tag=arm_tag) + if arm_tag == "left": + # Set specific position and orientation for left arm + target_pose[:2] = [-0.1, -0.05] + target_pose[2] -= 0.05 + target_pose[3:] = [-0.707, 0, -0.707, 0] + else: + # Set specific position and orientation for right arm + target_pose[:2] = [0.1, -0.05] + target_pose[2] -= 0.05 + target_pose[3:] = [0, 0.707, 0, -0.707] + + # Place the skillet to the defined target pose + self.move(self.move_to_pose(arm_tag=arm_tag, target_pose=target_pose)) + + # Get the functional point of the skillet as placement target for the bread + target_pose = self.skillet.get_functional_point(0) + + # Place the bread onto the skillet + self.move( + self.place_actor( + self.bread, + target_pose=target_pose, + arm_tag=arm_tag.opposite, + constrain="free", + pre_dis=0.05, + dis=0.05, + )) + + self.info["info"] = { + "{A}": f"106_skillet/base{self.skillet_id}", + "{B}": f"075_bread/base{self.bread_id}", + "{a}": str(arm_tag), + } + return self.info + + def check_success(self): + target_pose = self.skillet.get_functional_point(0) + bread_pose = self.bread.get_pose().p + return (np.all(abs(target_pose[:2] - bread_pose[:2]) < [0.035, 0.035]) + and target_pose[2] > 0.76 + self.table_z_bias and bread_pose[2] > 0.76 + self.table_z_bias) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_burger_fries.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_burger_fries.py new file mode 100644 index 00000000..16dc8227 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_burger_fries.py @@ -0,0 +1,131 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math +from ._GLOBAL_CONFIGS import * +from copy import deepcopy + + +class place_burger_fries(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + rand_pos_1 = rand_pose( + xlim=[-0.0, 0.0], + ylim=[-0.15, -0.1], + qpos=[0.706527, 0.706483, -0.0291356, -0.0291767], + rotate_rand=True, + rotate_lim=[0, 0, 0], + ) + self.tray_id = np.random.choice([0, 1, 2, 3], 1)[0] + self.tray = create_actor( + scene=self, + pose=rand_pos_1, + modelname="008_tray", + convex=True, + model_id=self.tray_id, + scale=(2.0, 2.0, 2.0), + is_static=True, + ) + self.tray.set_mass(0.05) + + rand_pos_2 = rand_pose( + xlim=[-0.3, -0.25], + ylim=[-0.15, -0.07], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, 0, 0], + ) + self.object1_id = np.random.choice([0, 1, 2, 3, 4, 5], 1)[0] + self.hamburg = create_actor( + scene=self, + pose=rand_pos_2, + modelname="006_hamburg", + convex=True, + model_id=self.object1_id, + ) + self.hamburg.set_mass(0.05) + + rand_pos_3 = rand_pose( + xlim=[0.2, 0.3], + ylim=[-0.15, -0.07], + qpos=[1.0, 0.0, 0.0, 0.0], + rotate_rand=True, + rotate_lim=[0, 0, 0], + ) + self.object2_id = np.random.choice([0, 1], 1)[0] + self.frenchfries = create_actor( + scene=self, + pose=rand_pos_3, + modelname="005_french-fries", + convex=True, + model_id=self.object2_id, + ) + self.frenchfries.set_mass(0.05) + + self.add_prohibit_area(self.tray, padding=0.1) + self.add_prohibit_area(self.hamburg, padding=0.05) + self.add_prohibit_area(self.frenchfries, padding=0.05) + + def play_once(self): + arm_tag_left = ArmTag("left") + arm_tag_right = ArmTag("right") + + # Dual grasp of hamburg and french fries + self.move( + self.grasp_actor(self.hamburg, arm_tag=arm_tag_left, pre_grasp_dis=0.1), + self.grasp_actor(self.frenchfries, arm_tag=arm_tag_right, pre_grasp_dis=0.1), + ) + + # Move up before placing + self.move( + self.move_by_displacement(arm_tag=arm_tag_left, z=0.1), + self.move_by_displacement(arm_tag=arm_tag_right, z=0.1), + ) + + # Get target poses from tray for placing + tray_place_pose_left = self.tray.get_functional_point(0) + tray_place_pose_right = self.tray.get_functional_point(1) + + # Place hamburg on tray + self.move( + self.place_actor(self.hamburg, + arm_tag=arm_tag_left, + target_pose=tray_place_pose_left, + functional_point_id=0, + constrain="free", + pre_dis=0.1, + pre_dis_axis='fp'), ) + + # Move up after placing + self.move(self.move_by_displacement(arm_tag=arm_tag_left, z=0.08), ) + + self.move( + self.place_actor(self.frenchfries, + arm_tag=arm_tag_right, + target_pose=tray_place_pose_right, + functional_point_id=0, + constrain="free", + pre_dis=0.1, + pre_dis_axis='fp'), + self.back_to_origin(arm_tag=arm_tag_left), + ) + + self.move(self.move_by_displacement(arm_tag=arm_tag_right, z=0.08)) + + self.info['info'] = { + "{A}": f"006_hamburg/base{self.object1_id}", + "{B}": f"008_tray/base{self.tray_id}", + "{C}": f"005_french-fries/base{self.object2_id}", + } + return self.info + + def check_success(self): + dis1 = np.linalg.norm( + self.tray.get_functional_point(0, "pose").p[0:2] - self.hamburg.get_functional_point(0, "pose").p[0:2]) + dis2 = np.linalg.norm( + self.tray.get_functional_point(1, "pose").p[0:2] - self.frenchfries.get_functional_point(0, "pose").p[0:2]) + threshold = 0.08 + return dis1 < threshold and dis2 < threshold and self.is_left_gripper_open() and self.is_right_gripper_open() diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_can_basket.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_can_basket.py new file mode 100644 index 00000000..9b5d73fd --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_can_basket.py @@ -0,0 +1,151 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math + + +class place_can_basket(Base_Task): + + def setup_demo(self, is_test=False, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + self.arm_tag = ArmTag({0: "left", 1: "right"}[np.random.randint(0, 2)]) + + self.basket_name = "110_basket" + self.basket_id = [0, 1][np.random.randint(0, 2)] + + can_dict = { + "071_can": [0, 1, 2, 3, 5, 6], + } + self.can_name = "071_can" + self.can_id = can_dict[self.can_name][np.random.randint(0, len(can_dict[self.can_name]))] + + if self.arm_tag == "left": # can on left + self.basket = rand_create_actor( + scene=self, + modelname=self.basket_name, + model_id=self.basket_id, + xlim=[0.02, 0.02], + ylim=[-0.08, -0.05], + qpos=[0.5, 0.5, 0.5, 0.5], + convex=True, + ) + self.can = rand_create_actor( + scene=self, + modelname=self.can_name, + model_id=self.can_id, + xlim=[-0.25, -0.2], + ylim=[0.0, 0.1], + qpos=[0.707225, 0.706849, -0.0100455, -0.00982061], + convex=True, + ) + else: # can on right + self.basket = rand_create_actor( + scene=self, + modelname=self.basket_name, + model_id=self.basket_id, + xlim=[-0.02, -0.02], + ylim=[-0.08, -0.05], + qpos=[0.5, 0.5, 0.5, 0.5], + convex=True, + ) + self.can = rand_create_actor( + scene=self, + modelname=self.can_name, + model_id=self.can_id, + xlim=[0.2, 0.25], + ylim=[0.0, 0.1], + qpos=[0.707225, 0.706849, -0.0100455, -0.00982061], + convex=True, + ) + self.start_height = self.basket.get_pose().p[2] + self.basket.set_mass(0.5) + self.can.set_mass(0.01) + self.add_prohibit_area(self.can, padding=0.1) + self.add_prohibit_area(self.basket, padding=0.05) + self.object_start_height = self.can.get_pose().p[2] + + def play_once(self): + # Grasp the can with the specified arm + self.move(self.grasp_actor(self.can, arm_tag=self.arm_tag, pre_grasp_dis=0.05)) + + # Determine the appropriate placement pose based on proximity to functional points of the basket + place_pose = self.get_arm_pose(arm_tag=self.arm_tag) + f0 = np.array(self.basket.get_functional_point(0)) + f1 = np.array(self.basket.get_functional_point(1)) + if np.linalg.norm(f0[:2] - place_pose[:2]) < np.linalg.norm(f1[:2] - place_pose[:2]): + place_pose = f0 + place_pose[:2] = f0[:2] + place_pose[3:] = ((-1, 0, 0, 0) if self.arm_tag == "left" else (0.05, 0, 0, 0.99)) + else: + place_pose = f1 + place_pose[:2] = f1[:2] + place_pose[3:] = ((-1, 0, 0, 0) if self.arm_tag == "left" else (0.05, 0, 0, 0.99)) + + # Place the can at the selected position into the basket + self.move( + self.place_actor( + self.can, + arm_tag=self.arm_tag, + target_pose=place_pose, + dis=0.02, + is_open=False, + constrain="free", + )) + + # If planning was not successful before, change to another posture to place the can + if self.plan_success is False: + self.plan_success = True # Try new way + + # slightly change the place pose + place_pose[0] += -0.15 if self.arm_tag == "left" else 0.15 + place_pose[2] += 0.15 + # Move arm to adjusted placement pose + self.move(self.move_to_pose(arm_tag=self.arm_tag, target_pose=place_pose)) + # Move down slightly + self.move(self.move_by_displacement(arm_tag=self.arm_tag, z=-0.1)) + # Open the gripper to release the can + self.move(self.open_gripper(arm_tag=self.arm_tag)) + # Return current arm to origin and grasp basket with opposite arm + self.move( + self.back_to_origin(arm_tag=self.arm_tag), + self.grasp_actor(self.basket, arm_tag=self.arm_tag.opposite, pre_grasp_dis=0.02), + ) + else: + # Open the gripper to release the can + self.move(self.open_gripper(arm_tag=self.arm_tag)) + # Move current arm upward to avoid collision + self.move(self.move_by_displacement(arm_tag=self.arm_tag, z=0.12)) + # Return current arm to origin and grasp basket with opposite arm + self.move( + self.back_to_origin(arm_tag=self.arm_tag), + self.grasp_actor(self.basket, arm_tag=self.arm_tag.opposite, pre_grasp_dis=0.08), + ) + + # Close the opposite arm's gripper to firmly grasp the basket + self.move(self.close_gripper(arm_tag=self.arm_tag.opposite)) + # Lift and slightly pull the basket inward + self.move( + self.move_by_displacement(arm_tag=self.arm_tag.opposite, + x=-0.02 if self.arm_tag.opposite == "left" else 0.02, + z=0.05)) + + self.info["info"] = { + "{A}": f"{self.can_name}/base{self.can_id}", + "{B}": f"{self.basket_name}/base{self.basket_id}", + "{a}": str(self.arm_tag), + } + return self.info + + def check_success(self): + can_p = self.can.get_pose().p + basket_p = self.basket.get_pose().p + basket_axis = (self.basket.get_pose().to_transformation_matrix()[:3, :3] @ np.array([[0, 1, 0]]).T) + can_contact_table = not self.check_actors_contact("071_can", "table") + can_contact_basket = self.check_actors_contact("071_can", "110_basket") + return (basket_p[2] - self.start_height > 0.02 and \ + can_p[2] - self.object_start_height > 0.02 and \ + np.dot(basket_axis.reshape(3), [0, 0, 1]) > 0.5 and \ + np.sum(np.sqrt(np.power(can_p - basket_p, 2))) < 0.15 and \ + can_contact_table and can_contact_basket) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_cans_plasticbox.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_cans_plasticbox.py new file mode 100644 index 00000000..7e6a3506 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_cans_plasticbox.py @@ -0,0 +1,135 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math +from ._GLOBAL_CONFIGS import * +from copy import deepcopy + + +class place_cans_plasticbox(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + rand_pos_1 = rand_pose( + xlim=[-0.0, 0.0], + ylim=[-0.15, -0.1], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, 0, 0], + ) + + self.plasticbox_id = np.random.choice([3, 5], 1)[0] + + self.plasticbox = create_actor( + scene=self, + pose=rand_pos_1, + modelname="062_plasticbox", + convex=True, + model_id=self.plasticbox_id, + ) + self.plasticbox.set_mass(0.05) + + rand_pos_2 = rand_pose( + xlim=[-0.25, -0.15], + ylim=[-0.15, -0.07], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, 0, 0], + ) + + self.object1_id = np.random.choice([0, 1, 2, 3, 5, 6], 1)[0] + + self.object1 = create_actor( + scene=self, + pose=rand_pos_2, + modelname="071_can", + convex=True, + model_id=self.object1_id, + ) + self.object1.set_mass(0.05) + + rand_pos_3 = rand_pose( + xlim=[0.15, 0.25], + ylim=[-0.15, -0.07], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, 0, 0], + ) + + self.object2_id = np.random.choice([0, 1, 2, 3, 5, 6], 1)[0] + + self.object2 = create_actor( + scene=self, + pose=rand_pos_3, + modelname="071_can", + convex=True, + model_id=self.object2_id, + ) + self.object2.set_mass(0.05) + + self.add_prohibit_area(self.plasticbox, padding=0.1) + self.add_prohibit_area(self.object1, padding=0.05) + self.add_prohibit_area(self.object2, padding=0.05) + + def play_once(self): + arm_tag_left = ArmTag("left") + arm_tag_right = ArmTag("right") + + # Grasp both objects with dual arms + self.move( + self.grasp_actor(self.object1, arm_tag=arm_tag_left, pre_grasp_dis=0.1), + self.grasp_actor(self.object2, arm_tag=arm_tag_right, pre_grasp_dis=0.1), + ) + + # Lift up both arms after grasping + self.move( + self.move_by_displacement(arm_tag=arm_tag_left, z=0.2), + self.move_by_displacement(arm_tag=arm_tag_right, z=0.2), + ) + + # Place left object into plastic box at target point 1 + self.move( + self.place_actor( + self.object1, + arm_tag=arm_tag_left, + target_pose=self.plasticbox.get_functional_point(1), + constrain="free", + pre_dis=0.1, + )) + + self.move(self.move_by_displacement(arm_tag=arm_tag_left, z=0.08)) + + # Left arm moves back to origin while right arm places object into plastic box at target point 0 + self.move( + self.back_to_origin(arm_tag=arm_tag_left), + self.place_actor( + self.object2, + arm_tag=arm_tag_right, + target_pose=self.plasticbox.get_functional_point(0), + constrain="free", + pre_dis=0.1, + ), + ) + + self.move(self.move_by_displacement(arm_tag=arm_tag_right, z=0.08)) + # Right arm moves back to original position + self.move(self.back_to_origin(arm_tag=arm_tag_right)) + + self.info["info"] = { + "{A}": f"071_can/base{self.object1_id}", + "{B}": f"062_plasticbox/base{self.plasticbox_id}", + "{C}": f"071_can/base{self.object2_id}", + } + return self.info + + def check_success(self): + plasticbox_functional_points_0 = self.plasticbox.get_functional_point(0)[0:2] + plasticbox_functional_points_1 = self.plasticbox.get_functional_point(1)[0:2] + dis1 = min(np.linalg.norm(self.object1.get_pose().p[0:2] - plasticbox_functional_points_0), + np.linalg.norm(self.object1.get_pose().p[0:2] - plasticbox_functional_points_1)) + dis2 = min(np.linalg.norm(self.object2.get_pose().p[0:2] - plasticbox_functional_points_0), + np.linalg.norm(self.object2.get_pose().p[0:2] - plasticbox_functional_points_1)) + threshold = 0.04 + return dis1 < threshold and dis2 < threshold and self.is_left_gripper_open() and self.is_right_gripper_open() \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_container_plate.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_container_plate.py new file mode 100755 index 00000000..4fdaff79 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_container_plate.py @@ -0,0 +1,98 @@ +from ._base_task import Base_Task +from .utils import * +import sapien + + +class place_container_plate(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + container_pose = rand_pose( + xlim=[-0.28, 0.28], + ylim=[-0.1, 0.05], + rotate_rand=False, + qpos=[0.5, 0.5, 0.5, 0.5], + ) + while abs(container_pose.p[0]) < 0.2: + container_pose = rand_pose( + xlim=[-0.28, 0.28], + ylim=[-0.1, 0.05], + rotate_rand=False, + qpos=[0.5, 0.5, 0.5, 0.5], + ) + id_list = {"002_bowl": [1, 2, 3, 5], "021_cup": [1, 2, 3, 4, 5, 6, 7]} + self.actor_name = np.random.choice(["002_bowl", "021_cup"]) + self.container_id = np.random.choice(id_list[self.actor_name]) + self.container = create_actor( + self, + pose=container_pose, + modelname=self.actor_name, + model_id=self.container_id, + convex=True, + ) + + x = 0.05 if self.container.get_pose().p[0] > 0 else -0.05 + self.plate_id = 0 + pose = rand_pose( + xlim=[x - 0.03, x + 0.03], + ylim=[-0.15, -0.1], + rotate_rand=False, + qpos=[0.5, 0.5, 0.5, 0.5], + ) + self.plate = create_actor( + self, + pose=pose, + modelname="003_plate", + scale=[0.025, 0.025, 0.025], + is_static=True, + convex=True, + ) + self.add_prohibit_area(self.container, padding=0.1) + self.add_prohibit_area(self.plate, padding=0.1) + + def play_once(self): + # Get container's position to determine which arm to use + container_pose = self.container.get_pose().p + # Select arm based on container's x position (right if positive, left if negative) + arm_tag = ArmTag("right" if container_pose[0] > 0 else "left") + + # Grasp the container using selected arm with specific contact point + self.move( + self.grasp_actor( + self.container, + arm_tag=arm_tag, + contact_point_id=[0, 2][int(arm_tag == "left")], + pre_grasp_dis=0.1, + )) + # Lift the container up by 0.1m along z-axis + self.move(self.move_by_displacement(arm_tag, z=0.1, move_axis="arm")) + + # Place the container onto the plate's functional point + self.move( + self.place_actor( + self.container, + target_pose=self.plate.get_functional_point(0), + arm_tag=arm_tag, + functional_point_id=0, + pre_dis=0.12, + dis=0.03, + )) + # Move the arm up by 0.1m after placing + self.move(self.move_by_displacement(arm_tag, z=0.08, move_axis="arm")) + + # Record information about the objects and arm used + self.info["info"] = { + "{A}": f"003_plate/base{self.plate_id}", + "{B}": f"{self.actor_name}/base{self.container_id}", + "{a}": str(arm_tag), + } + return self.info + + def check_success(self): + container_pose = self.container.get_pose().p + target_pose = self.plate.get_pose().p + eps = np.array([0.05, 0.05, 0.03]) + return (np.all(abs(container_pose[:3] - target_pose) < eps) and self.is_left_gripper_open() + and self.is_right_gripper_open()) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_dual_shoes.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_dual_shoes.py new file mode 100644 index 00000000..3bf12147 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_dual_shoes.py @@ -0,0 +1,159 @@ +from ._base_task import Base_Task +from .utils import * +import math +import sapien +from ._GLOBAL_CONFIGS import * + + +class place_dual_shoes(Base_Task): + + def setup_demo(self, is_test=False, **kwags): + super()._init_task_env_(table_height_bias=-0.1, **kwags) + + def load_actors(self): + self.shoe_box = create_actor( + self, + pose=sapien.Pose([0, -0.13, 0.74], [0.5, 0.5, -0.5, -0.5]), + modelname="007_shoe-box", + convex=True, + is_static=True, + ) + + shoe_id = np.random.choice([i for i in range(10)]) + self.shoe_id = shoe_id + + # left shoe + shoes_pose = rand_pose( + xlim=[-0.3, -0.2], + ylim=[-0.1, 0.05], + zlim=[0.741], + ylim_prop=True, + rotate_rand=True, + rotate_lim=[0, 3.14, 0], + qpos=[0.707, 0.707, 0, 0], + ) + + while np.sum(pow(shoes_pose.get_p()[:2] - np.zeros(2), 2)) < 0.0225: + shoes_pose = rand_pose( + xlim=[-0.3, -0.2], + ylim=[-0.1, 0.05], + zlim=[0.741], + ylim_prop=True, + rotate_rand=True, + rotate_lim=[0, 3.14, 0], + qpos=[0.707, 0.707, 0, 0], + ) + + self.left_shoe = create_actor( + self, + pose=shoes_pose, + modelname="041_shoe", + convex=True, + model_id=shoe_id, + ) + + # right shoe + shoes_pose = rand_pose( + xlim=[0.2, 0.3], + ylim=[-0.1, 0.05], + zlim=[0.741], + ylim_prop=True, + rotate_rand=True, + rotate_lim=[0, 3.14, 0], + qpos=[0.707, 0.707, 0, 0], + ) + + while np.sum(pow(shoes_pose.get_p()[:2] - np.zeros(2), 2)) < 0.0225: + shoes_pose = rand_pose( + xlim=[0.2, 0.3], + ylim=[-0.1, 0.05], + zlim=[0.741], + ylim_prop=True, + rotate_rand=True, + rotate_lim=[0, 3.14, 0], + qpos=[0.707, 0.707, 0, 0], + ) + + self.right_shoe = create_actor( + self, + pose=shoes_pose, + modelname="041_shoe", + convex=True, + model_id=shoe_id, + ) + + self.add_prohibit_area(self.left_shoe, padding=0.02) + self.add_prohibit_area(self.right_shoe, padding=0.02) + self.prohibited_area.append([-0.15, -0.25, 0.15, 0.01]) + self.right_shoe_middle_pose = [0.35, -0.05, 0.79, 0, 1, 0, 0] + + def play_once(self): + left_arm_tag = ArmTag("left") + right_arm_tag = ArmTag("right") + # Grasp both left and right shoes simultaneously + self.move( + self.grasp_actor(self.left_shoe, arm_tag=left_arm_tag, pre_grasp_dis=0.1), + self.grasp_actor(self.right_shoe, arm_tag=right_arm_tag, pre_grasp_dis=0.1), + ) + # Lift both shoes up simultaneously + self.move( + self.move_by_displacement(left_arm_tag, z=0.15), + self.move_by_displacement(right_arm_tag, z=0.15), + ) + # Get target positions for placing shoes in the shoe box + left_target = self.shoe_box.get_functional_point(0) + right_target = self.shoe_box.get_functional_point(1) + # Prepare place actions for both shoes + left_place_pose = self.place_actor( + self.left_shoe, + target_pose=left_target, + arm_tag=left_arm_tag, + functional_point_id=0, + pre_dis=0.07, + dis=0.02, + constrain="align", + ) + right_place_pose = self.place_actor( + self.right_shoe, + target_pose=right_target, + arm_tag=right_arm_tag, + functional_point_id=0, + pre_dis=0.07, + dis=0.02, + constrain="align", + ) + # Place left shoe while moving right arm to prepare for placement + self.move( + left_place_pose, + self.move_by_displacement(right_arm_tag, x=0.1, y=-0.05, quat=GRASP_DIRECTION_DIC["top_down"]), + ) + # Return left arm to origin while placing right shoe + self.move(self.back_to_origin(left_arm_tag), right_place_pose) + + self.delay(3) + + self.info["info"] = { + "{A}": f"041_shoe/base{self.shoe_id}", + "{B}": f"007_shoe-box/base0", + } + return self.info + + def check_success(self): + left_shoe_pose_p = np.array(self.left_shoe.get_pose().p) + left_shoe_pose_q = np.array(self.left_shoe.get_pose().q) + right_shoe_pose_p = np.array(self.right_shoe.get_pose().p) + right_shoe_pose_q = np.array(self.right_shoe.get_pose().q) + if left_shoe_pose_q[0] < 0: + left_shoe_pose_q *= -1 + if right_shoe_pose_q[0] < 0: + right_shoe_pose_q *= -1 + target_pose_p = np.array([0, -0.13]) + target_pose_q = np.array([0.5, 0.5, -0.5, -0.5]) + eps = np.array([0.05, 0.05, 0.07, 0.07, 0.07, 0.07]) + return (np.all(abs(left_shoe_pose_p[:2] - (target_pose_p - [0, 0.04])) < eps[:2]) + and np.all(abs(left_shoe_pose_q - target_pose_q) < eps[-4:]) + and np.all(abs(right_shoe_pose_p[:2] - (target_pose_p + [0, 0.04])) < eps[:2]) + and np.all(abs(right_shoe_pose_q - target_pose_q) < eps[-4:]) + and abs(left_shoe_pose_p[2] - (self.shoe_box.get_pose().p[2] + 0.01)) < 0.03 + and abs(right_shoe_pose_p[2] - (self.shoe_box.get_pose().p[2] + 0.01)) < 0.03 + and self.is_left_gripper_open() and self.is_right_gripper_open()) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_empty_cup.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_empty_cup.py new file mode 100755 index 00000000..811c9e4a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_empty_cup.py @@ -0,0 +1,98 @@ +from ._base_task import Base_Task +from .utils import * +import sapien + + +class place_empty_cup(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + tag = np.random.randint(0, 2) + cup_xlim = [[0.15, 0.3], [-0.3, -0.15]] + coaster_lim = [[-0.05, 0.1], [-0.1, 0.05]] + self.cup = rand_create_actor( + self, + xlim=cup_xlim[tag], + ylim=[-0.2, 0.05], + modelname="021_cup", + rotate_rand=False, + qpos=[0.5, 0.5, 0.5, 0.5], + convex=True, + model_id=0, + ) + cup_pose = self.cup.get_pose().p + + coaster_pose = rand_pose( + xlim=coaster_lim[tag], + ylim=[-0.2, 0.05], + rotate_rand=False, + qpos=[0.5, 0.5, 0.5, 0.5], + ) + + while np.sum(pow(cup_pose[:2] - coaster_pose.p[:2], 2)) < 0.01: + coaster_pose = rand_pose( + xlim=coaster_lim[tag], + ylim=[-0.2, 0.05], + rotate_rand=False, + qpos=[0.5, 0.5, 0.5, 0.5], + ) + self.coaster = create_actor( + self, + pose=coaster_pose, + modelname="019_coaster", + convex=True, + model_id=0, + is_static=True + ) + + self.add_prohibit_area(self.cup, padding=0.05) + self.add_prohibit_area(self.coaster, padding=0.05) + self.delay(2) + cup_pose = self.cup.get_pose().p + + def play_once(self): + # Get the current pose of the cup + cup_pose = self.cup.get_pose().p + # Determine which arm to use based on cup's x position (right if positive, left if negative) + arm_tag = ArmTag("right" if cup_pose[0] > 0 else "left") + + # Close the gripper to prepare for grasping + self.move(self.close_gripper(arm_tag, pos=0.6)) + # Grasp the cup using the selected arm + self.move( + self.grasp_actor( + self.cup, + arm_tag, + pre_grasp_dis=0.1, + contact_point_id=[0, 2][int(arm_tag == "left")], + )) + # Lift the cup up by 0.08 meters along z-axis + self.move(self.move_by_displacement(arm_tag, z=0.08, move_axis="arm")) + + # Get coaster's functional point as target pose + target_pose = self.coaster.get_functional_point(0) + # Place the cup onto the coaster + self.move(self.place_actor( + self.cup, + arm_tag, + target_pose=target_pose, + functional_point_id=0, + pre_dis=0.05, + )) + # Lift the arm slightly (0.05m) after placing to avoid collision + self.move(self.move_by_displacement(arm_tag, z=0.05, move_axis="arm")) + + self.info["info"] = {"{A}": "021_cup/base0", "{B}": "019_coaster/base0"} + return self.info + + def check_success(self): + # eps = [0.03, 0.03, 0.015] + eps = 0.035 + cup_pose = self.cup.get_functional_point(0, "pose").p + coaster_pose = self.coaster.get_functional_point(0, "pose").p + return ( + # np.all(np.abs(cup_pose - coaster_pose) < eps) + np.sum(pow(cup_pose[:2] - coaster_pose[:2], 2)) < eps**2 and abs(cup_pose[2] - coaster_pose[2]) < 0.015 + and self.is_left_gripper_open() and self.is_right_gripper_open()) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_fan.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_fan.py new file mode 100644 index 00000000..ee8d0b53 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_fan.py @@ -0,0 +1,129 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math +from copy import deepcopy +import numpy as np + + +class place_fan(Base_Task): + + def setup_demo(self, is_test=False, **kwargs): + super()._init_task_env_(**kwargs) + + def load_actors(self): + rand_pos = rand_pose( + xlim=[-0.1, 0.1], + ylim=[-0.15, -0.05], + qpos=[0.0, 0.0, 0.707, 0.707], + rotate_rand=True, + rotate_lim=[0, 2 * np.pi, 0], + ) + id_list = [4, 5] + self.fan_id = np.random.choice(id_list) + self.fan = create_actor( + scene=self, + pose=rand_pos, + modelname="099_fan", + convex=True, + model_id=self.fan_id, + ) + self.fan.set_mass(0.01) + + xlim = [0.15, 0.25] if self.fan.get_pose().p[0] > 0 else [-0.25, -0.15] + rand_pos = rand_pose( + xlim=xlim, + ylim=[-0.15, -0.05], + ) + + colors = { + "Red": (1, 0, 0), + "Green": (0, 1, 0), + "Blue": (0, 0, 1), + "Yellow": (1, 1, 0), + "Cyan": (0, 1, 1), + "Magenta": (1, 0, 1), + "Black": (0, 0, 0), + "Gray": (0.5, 0.5, 0.5), + "Orange": (1, 0.5, 0), + "Purple": (0.5, 0, 0.5), + "Brown": (0.65, 0.4, 0.16), + "Pink": (1, 0.75, 0.8), + "Lime": (0.5, 1, 0), + "Olive": (0.5, 0.5, 0), + "Teal": (0, 0.5, 0.5), + "Maroon": (0.5, 0, 0), + "Navy": (0, 0, 0.5), + "Coral": (1, 0.5, 0.31), + "Turquoise": (0.25, 0.88, 0.82), + "Indigo": (0.29, 0, 0.51), + "Beige": (0.96, 0.91, 0.81), + "Tan": (0.82, 0.71, 0.55), + "Silver": (0.75, 0.75, 0.75), + } + + color_items = list(colors.items()) + idx = np.random.choice(len(color_items)) + self.color_name, self.color_value = color_items[idx] + + self.pad = create_box( + scene=self.scene, + pose=rand_pos, + half_size=(0.05, 0.05, 0.001), + color=self.color_value, + name="box", + ) + + self.pad.set_mass(1) + self.add_prohibit_area(self.fan, padding=0.07) + self.prohibited_area.append([ + rand_pos.p[0] - 0.15, + rand_pos.p[1] - 0.15, + rand_pos.p[0] + 0.15, + rand_pos.p[1] + 0.15, + ]) + # Get the target pose for placing the fan from the pad's current pose + target_pose = self.pad.get_pose().p + self.target_pose = target_pose.tolist() + [1, 0, 0, 0] + + def play_once(self): + # Determine which arm is closer to the object based on x-coordinate of the fan's position + arm_tag = ArmTag("right" if self.fan.get_pose().p[0] > 0 else "left") + + # Grasp the fan with the selected arm + self.move(self.grasp_actor(self.fan, arm_tag=arm_tag, pre_grasp_dis=0.05)) + # Lift the fan slightly after grasping + self.move(self.move_by_displacement(arm_tag=arm_tag, z=0.05)) + + # Place the fan onto the pad with alignment constraint along specified axes + self.move( + self.place_actor( + self.fan, + arm_tag=arm_tag, + target_pose=self.target_pose, + constrain="align", + pre_dis=0.04, + dis=0.005, + )) + + self.info["info"] = { + "{A}": f"099_fan/base{self.fan_id}", + "{B}": self.color_name, + "{a}": str(arm_tag), + } + return self.info + + def check_success(self): + fan_qpose = self.fan.get_pose().q + fan_pose = self.fan.get_pose().p + + target_pose = self.target_pose[:3] + target_qpose = np.array([0.707, 0.707, 0.0, 0.0]) + + if fan_qpose[0] < 0: + fan_qpose *= -1 + + eps = np.array([0.05, 0.05, 0.05, 0.05]) + + return (np.all(abs(fan_qpose - target_qpose) < eps[-4:]) and self.robot.is_left_gripper_open() + and self.robot.is_right_gripper_open()) and (np.all(abs(fan_pose - target_pose) < np.array([0.04, 0.04, 0.04]))) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_mouse_pad.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_mouse_pad.py new file mode 100644 index 00000000..3cdfbfae --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_mouse_pad.py @@ -0,0 +1,128 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math +from ._GLOBAL_CONFIGS import * +from copy import deepcopy +import numpy as np + + +class place_mouse_pad(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + rand_pos = rand_pose( + xlim=[-0.25, 0.25], + ylim=[-0.2, 0.0], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, 3.14, 0], + ) + while abs(rand_pos.p[0]) < 0.05: + rand_pos = rand_pose( + xlim=[-0.25, 0.25], + ylim=[-0.2, 0.0], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, np.pi / 4, 0], + ) + + self.mouse_id = np.random.choice([0, 1, 2], 1)[0] + self.mouse = create_actor( + scene=self, + pose=rand_pos, + modelname="047_mouse", + convex=True, + model_id=self.mouse_id, + ) + self.mouse.set_mass(0.05) + + if rand_pos.p[0] > 0: + xlim = [0.05, 0.25] + else: + xlim = [-0.25, -0.05] + target_rand_pose = rand_pose( + xlim=xlim, + ylim=[-0.2, 0.0], + qpos=[1, 0, 0, 0], + rotate_rand=False, + ) + while (np.sqrt((target_rand_pose.p[0] - rand_pos.p[0])**2 + (target_rand_pose.p[1] - rand_pos.p[1])**2) < 0.1): + target_rand_pose = rand_pose( + xlim=xlim, + ylim=[-0.2, 0.0], + qpos=[1, 0, 0, 0], + rotate_rand=False, + ) + + colors = { + "Red": (1, 0, 0), + "Green": (0, 1, 0), + "Blue": (0, 0, 1), + "Yellow": (1, 1, 0), + "Cyan": (0, 1, 1), + "Magenta": (1, 0, 1), + "Black": (0, 0, 0), + "Gray": (0.5, 0.5, 0.5), + } + + color_items = list(colors.items()) + color_index = np.random.choice(len(color_items)) + self.color_name, self.color_value = color_items[color_index] + + half_size = [0.035, 0.065, 0.0005] + self.target = create_box( + scene=self, + pose=target_rand_pose, + half_size=half_size, + color=self.color_value, + name="box", + is_static=True, + ) + self.add_prohibit_area(self.target, padding=0.12) + self.add_prohibit_area(self.mouse, padding=0.03) + # Construct target pose with position from target object and identity orientation + self.target_pose = self.target.get_pose().p.tolist() + [0, 0, 0, 1] + + def play_once(self): + # Determine which arm to use based on mouse position (right if on right side, left otherwise) + arm_tag = ArmTag("right" if self.mouse.get_pose().p[0] > 0 else "left") + + # Grasp the mouse with the selected arm + self.move(self.grasp_actor(self.mouse, arm_tag=arm_tag, pre_grasp_dis=0.1)) + + # Lift the mouse upward by 0.1 meters in z-direction + self.move(self.move_by_displacement(arm_tag=arm_tag, z=0.1)) + + # Place the mouse at the target location with alignment constraint + self.move( + self.place_actor( + self.mouse, + arm_tag=arm_tag, + target_pose=self.target_pose, + constrain="align", + pre_dis=0.07, + dis=0.005, + )) + + # Record information about the objects and arm used in the task + self.info["info"] = { + "{A}": f"047_mouse/base{self.mouse_id}", + "{B}": f"{self.color_name}", + "{a}": str(arm_tag), + } + return self.info + + def check_success(self): + mouse_pose = self.mouse.get_pose().p + mouse_qpose = np.abs(self.mouse.get_pose().q) + target_pos = self.target.get_pose().p + eps1 = 0.015 + eps2 = 0.012 + + return (np.all(abs(mouse_pose[:2] - target_pos[:2]) < np.array([eps1, eps2])) + and (np.abs(mouse_qpose[2] * mouse_qpose[3] - 0.49) < eps1 + or np.abs(mouse_qpose[0] * mouse_qpose[1] - 0.49) < eps1) and self.robot.is_left_gripper_open() + and self.robot.is_right_gripper_open()) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_object_basket.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_object_basket.py new file mode 100644 index 00000000..9dd71029 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_object_basket.py @@ -0,0 +1,151 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math + + +class place_object_basket(Base_Task): + + def setup_demo(self, is_test=False, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + self.arm_tag = ArmTag({0: "left", 1: "right"}[np.random.randint(0, 2)]) + self.basket_name = "110_basket" + self.basket_id = np.random.randint(0, 2) + toycar_dict = { + "081_playingcards": [0, 1, 2], + "057_toycar": [0, 1, 2, 3, 4, 5], + } + self.object_name = ["081_playingcards", "057_toycar"][np.random.randint(0, 2)] + self.object_id = toycar_dict[self.object_name][np.random.randint(0, len(toycar_dict[self.object_name]))] + if self.arm_tag == "left": # toycar on left + self.basket = rand_create_actor( + scene=self, + modelname=self.basket_name, + model_id=self.basket_id, + xlim=[0.02, 0.02], + ylim=[-0.08, -0.05], + qpos=[0.5, 0.5, 0.5, 0.5], + convex=True, + ) + self.object = rand_create_actor( + scene=self, + modelname=self.object_name, + model_id=self.object_id, + xlim=[-0.25, -0.2], + ylim=[-0.1, 0.1], + rotate_rand=True, + rotate_lim=[0, np.pi / 6, 0], + qpos=[0.707225, 0.706849, -0.0100455, -0.00982061], + convex=True, + ) + else: # toycar on right + self.basket = rand_create_actor( + scene=self, + modelname=self.basket_name, + model_id=self.basket_id, + xlim=[-0.02, -0.02], + ylim=[-0.08, -0.05], + qpos=[0.5, 0.5, 0.5, 0.5], + convex=True, + ) + self.object = rand_create_actor( + scene=self, + modelname=self.object_name, + model_id=self.object_id, + xlim=[0.2, 0.25], + ylim=[-0.1, 0.1], + rotate_rand=True, + rotate_lim=[0, np.pi / 6, 0], + qpos=[0.707225, 0.706849, -0.0100455, -0.00982061], + convex=True, + ) + self.basket.set_mass(0.5) + self.object.set_mass(0.01) + self.object_start_height = self.object.get_pose().p[2] + self.start_height = self.basket.get_pose().p[2] + self.add_prohibit_area(self.object, padding=0.1) + self.add_prohibit_area(self.basket, padding=0.05) + + def play_once(self): + # Grasp the toy car + self.move(self.grasp_actor(self.object, arm_tag=self.arm_tag)) + + # Lift the toy car up + self.move(self.move_by_displacement(arm_tag=self.arm_tag, z=0.15)) + + # Get functional points of basket for placing + f0 = np.array(self.basket.get_functional_point(0)) + f1 = np.array(self.basket.get_functional_point(1)) + place_pose = (f0 if np.linalg.norm(f0[:2] - self.object.get_pose().p[:2]) + < np.linalg.norm(f1[:2] - self.object.get_pose().p[:2]) else f1) + place_pose[:2] = f0[:2] if place_pose is f0 else f1[:2] + place_pose[3:] = (-1, 0, 0, 0) if self.arm_tag == "left" else (0.05, 0, 0, 0.99) + + # Place the toy car in the basket + self.move(self.place_actor( + self.object, + arm_tag=self.arm_tag, + target_pose=place_pose, + dis=0.02, + is_open=False, + )) + + if not self.plan_success: + self.plan_success = True # Try new way + # Move up and away (recovery motion when plan fails) + place_pose[0] += -0.15 if self.arm_tag == "left" else 0.15 + place_pose[2] += 0.15 + self.move(self.move_to_pose(arm_tag=self.arm_tag, target_pose=place_pose)) + + # Lower down (recovery motion when plan fails) + place_pose[2] -= 0.05 + self.move(self.move_to_pose(arm_tag=self.arm_tag, target_pose=place_pose)) + + # Open gripper to release object + self.move(self.open_gripper(arm_tag=self.arm_tag)) + + # Move arm away and grasp the basket with opposite arm (recovery strategy) + self.move( + self.back_to_origin(arm_tag=self.arm_tag), + self.grasp_actor(self.basket, arm_tag=self.arm_tag.opposite, pre_grasp_dis=0.02), + ) + else: + # Open gripper to release object + self.move(self.open_gripper(arm_tag=self.arm_tag)) + # lift arm up, to avoid collision with the basket + self.move(self.move_by_displacement(arm_tag=self.arm_tag, z=0.08)) + # Move arm away and grasp the basket with opposite arm + self.move( + self.back_to_origin(arm_tag=self.arm_tag), + self.grasp_actor(self.basket, arm_tag=self.arm_tag.opposite, pre_grasp_dis=0.08), + ) + + # Lift basket a bit after grasping + self.move( + self.move_by_displacement( + arm_tag=self.arm_tag.opposite, + x=0.05 if self.arm_tag.opposite == "right" else -0.05, + z=0.05, + )) + + self.info["info"] = { + "{A}": f"{self.object_name}/base{self.object_id}", + "{B}": f"{self.basket_name}/base{self.basket_id}", + "{a}": str(self.arm_tag), + "{b}": str(self.arm_tag.opposite), + } + return self.info + + def check_success(self): + toy_p = self.object.get_pose().p + basket_p = self.basket.get_pose().p + basket_axis = (self.basket.get_pose().to_transformation_matrix()[:3, :3] @ np.array([[0, 1, 0]]).T) + obj_contact_table = not self.check_actors_contact(self.object_name, "table") + obj_contact_basket = self.check_actors_contact(self.object_name, self.basket_name) + return (basket_p[2] - self.start_height > 0.02 and \ + toy_p[2] - self.object_start_height > 0.02 and \ + np.dot(basket_axis.reshape(3), [0, 0, 1]) > 0.5 and \ + np.sum(np.sqrt((toy_p - basket_p)**2)) < 0.15 and \ + obj_contact_table and obj_contact_basket) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_object_scale.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_object_scale.py new file mode 100644 index 00000000..6887282e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_object_scale.py @@ -0,0 +1,136 @@ +from copy import deepcopy +from ._base_task import Base_Task +from .utils import * +import sapien +import math +import glob +import numpy as np + + +class place_object_scale(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + rand_pos = rand_pose( + xlim=[-0.25, 0.25], + ylim=[-0.2, 0.05], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, 3.14, 0], + ) + while abs(rand_pos.p[0]) < 0.02: + rand_pos = rand_pose( + xlim=[-0.25, 0.25], + ylim=[-0.2, 0.05], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, 3.14, 0], + ) + + def get_available_model_ids(modelname): + asset_path = os.path.join("assets/objects", modelname) + json_files = glob.glob(os.path.join(asset_path, "model_data*.json")) + + available_ids = [] + for file in json_files: + base = os.path.basename(file) + try: + idx = int(base.replace("model_data", "").replace(".json", "")) + available_ids.append(idx) + except ValueError: + continue + + return available_ids + + object_list = ["047_mouse", "048_stapler", "050_bell"] + + self.selected_modelname = np.random.choice(object_list) + + available_model_ids = get_available_model_ids(self.selected_modelname) + if not available_model_ids: + raise ValueError(f"No available model_data.json files found for {self.selected_modelname}") + + self.selected_model_id = np.random.choice(available_model_ids) + + self.object = create_actor( + scene=self, + pose=rand_pos, + modelname=self.selected_modelname, + convex=True, + model_id=self.selected_model_id, + ) + self.object.set_mass(0.05) + + if rand_pos.p[0] > 0: + xlim = [0.02, 0.25] + else: + xlim = [-0.25, -0.02] + target_rand_pose = rand_pose( + xlim=xlim, + ylim=[-0.2, 0.05], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, 3.14, 0], + ) + while (np.sqrt((target_rand_pose.p[0] - rand_pos.p[0])**2 + (target_rand_pose.p[1] - rand_pos.p[1])**2) < 0.15): + target_rand_pose = rand_pose( + xlim=xlim, + ylim=[-0.2, 0.05], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, 3.14, 0], + ) + + self.scale_id = np.random.choice([0, 1, 5, 6], 1)[0] + + self.scale = create_actor( + scene=self, + pose=target_rand_pose, + modelname="072_electronicscale", + model_id=self.scale_id, + convex=True, + ) + self.scale.set_mass(0.05) + + self.add_prohibit_area(self.object, padding=0.05) + self.add_prohibit_area(self.scale, padding=0.05) + + def play_once(self): + # Determine which arm to use based on object's x position (right if positive, left if negative) + self.arm_tag = ArmTag("right" if self.object.get_pose().p[0] > 0 else "left") + + # Grasp the object with the selected arm + self.move(self.grasp_actor(self.object, arm_tag=self.arm_tag)) + + # Lift the object up by 0.15 meters in z-axis + self.move(self.move_by_displacement(arm_tag=self.arm_tag, z=0.15)) + + # Place the object on the scale's functional point with free constraint, + # using pre-placement distance of 0.05m and final placement distance of 0.005m + self.move( + self.place_actor( + self.object, + arm_tag=self.arm_tag, + target_pose=self.scale.get_functional_point(0), + constrain="free", + pre_dis=0.05, + dis=0.005, + )) + + # Record information about the objects and arm used for the task + self.info["info"] = { + "{A}": f"072_electronicscale/base{self.scale_id}", + "{B}": f"{self.selected_modelname}/base{self.selected_model_id}", + "{a}": str(self.arm_tag), + } + return self.info + + def check_success(self): + object_pose = self.object.get_pose().p + scale_pose = self.scale.get_functional_point(0) + distance_threshold = 0.035 + distance = np.linalg.norm(np.array(scale_pose[:2]) - np.array(object_pose[:2])) + check_arm = (self.is_left_gripper_open if self.arm_tag == "left" else self.is_right_gripper_open) + return (distance < distance_threshold and object_pose[2] > (scale_pose[2] - 0.01) and check_arm()) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_object_stand.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_object_stand.py new file mode 100644 index 00000000..11d8fca6 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_object_stand.py @@ -0,0 +1,139 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math +import glob +from copy import deepcopy + + +class place_object_stand(Base_Task): + + def setup_demo(self, is_test=False, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + rand_pos = rand_pose( + xlim=[-0.28, 0.28], + ylim=[-0.05, 0.05], + qpos=[0.707, 0.707, 0.0, 0.0], + rotate_rand=True, + rotate_lim=[0, np.pi / 3, 0], + ) + while abs(rand_pos.p[0]) < 0.2: + rand_pos = rand_pose( + xlim=[-0.28, 0.28], + ylim=[-0.05, 0.05], + qpos=[0.707, 0.707, 0.0, 0.0], + rotate_rand=True, + rotate_lim=[0, np.pi / 3, 0], + ) + + def get_available_model_ids(modelname): + asset_path = os.path.join("assets/objects", modelname) + json_files = glob.glob(os.path.join(asset_path, "model_data*.json")) + + available_ids = [] + for file in json_files: + base = os.path.basename(file) + try: + idx = int(base.replace("model_data", "").replace(".json", "")) + available_ids.append(idx) + except ValueError: + continue + + return available_ids + + object_list = [ + "047_mouse", + "048_stapler", + "050_bell", + "073_rubikscube", + "057_toycar", + "079_remotecontrol", + ] + self.selected_modelname = np.random.choice(object_list) + available_model_ids = get_available_model_ids(self.selected_modelname) + if not available_model_ids: + raise ValueError(f"No available model_data.json files found for {self.selected_modelname}") + self.selected_model_id = np.random.choice(available_model_ids) + self.object = create_actor( + scene=self, + pose=rand_pos, + modelname=self.selected_modelname, + convex=True, + model_id=self.selected_model_id, + ) + self.object.set_mass(0.05) + + object_pos = self.object.get_pose() + if object_pos.p[0] > 0: + xlim = [0.0, 0.05] + else: + xlim = [-0.05, 0.0] + target_rand_pos = rand_pose( + xlim=xlim, + ylim=[-0.15, -0.1], + qpos=[0.707, 0.707, 0.0, 0.0], + rotate_rand=True, + rotate_lim=[0, np.pi / 6, 0], + ) + while ((object_pos.p[0] - target_rand_pos.p[0])**2 + (object_pos.p[1] - target_rand_pos.p[1])**2) < 0.01: + target_rand_pos = rand_pose( + xlim=xlim, + ylim=[-0.15, -0.1], + qpos=[0.707, 0.707, 0.0, 0.0], + rotate_rand=True, + rotate_lim=[0, np.pi / 6, 0], + ) + id_list = [0, 1, 2, 3, 4] + self.displaystand_id = np.random.choice(id_list) + self.displaystand = create_actor( + scene=self, + pose=target_rand_pos, + modelname="074_displaystand", + convex=True, + model_id=self.displaystand_id, + ) + + self.object.set_mass(0.01) + self.displaystand.set_mass(0.01) + + self.add_prohibit_area(self.displaystand, padding=0.05) + self.add_prohibit_area(self.object, padding=0.1) + + def play_once(self): + # Determine which arm to use based on object's x position + arm_tag = ArmTag("right" if self.object.get_pose().p[0] > 0 else "left") + + # Grasp the object with specified arm + self.move(self.grasp_actor(self.object, arm_tag=arm_tag, pre_grasp_dis=0.1)) + # Lift the object up by 0.06 meters in z-direction + self.move(self.move_by_displacement(arm_tag=arm_tag, z=0.06)) + + # Get the target pose from display stand's functional point + displaystand_pose = self.displaystand.get_functional_point(0) + + # Place the object onto the display stand with free constraint + self.move( + self.place_actor( + self.object, + arm_tag=arm_tag, + target_pose=displaystand_pose, + constrain="free", + pre_dis=0.07, + )) + + # Store information about the objects and arm used in the info dictionary + self.info["info"] = { + "{A}": f"{self.selected_modelname}/base{self.selected_model_id}", + "{B}": f"074_displaystand/base{self.displaystand_id}", + "{a}": str(arm_tag), + } + return self.info + + def check_success(self): + object_pose = self.object.get_pose().p + displaystand_pose = self.displaystand.get_pose().p + eps1 = 0.03 + return (np.all(abs(object_pose[:2] - displaystand_pose[:2]) < np.array([eps1, eps1])) + and self.robot.is_left_gripper_open() and self.robot.is_right_gripper_open()) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_phone_stand.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_phone_stand.py new file mode 100644 index 00000000..fb0cab7c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_phone_stand.py @@ -0,0 +1,104 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +from copy import deepcopy + + +class place_phone_stand(Base_Task): + + def setup_demo(self, is_test=False, **kwargs): + super()._init_task_env_(**kwargs) + + def load_actors(self): + tag = np.random.randint(2) + ori_quat = [ + [0.707, 0.707, 0, 0], + [0.5, 0.5, 0.5, 0.5], + [0.5, 0.5, -0.5, -0.5], + [0.5, 0.5, -0.5, -0.5], + [0.5, -0.5, 0.5, -0.5], + ] + if tag == 0: + phone_x_lim = [-0.25, -0.05] + stand_x_lim = [-0.15, 0.0] + else: + phone_x_lim = [0.05, 0.25] + stand_x_lim = [0, 0.15] + + self.phone_id = np.random.choice([0, 1, 2, 4], 1)[0] + phone_pose = rand_pose( + xlim=phone_x_lim, + ylim=[-0.2, 0.0], + qpos=ori_quat[self.phone_id], + rotate_rand=True, + rotate_lim=[0, 0.7, 0], + ) + self.phone = create_actor( + scene=self, + pose=phone_pose, + modelname="077_phone", + convex=True, + model_id=self.phone_id, + ) + self.phone.set_mass(0.01) + + stand_pose = rand_pose( + xlim=stand_x_lim, + ylim=[0, 0.2], + qpos=[0.707, 0.707, 0, 0], + rotate_rand=False, + ) + while np.sqrt(np.sum((phone_pose.p[:2] - stand_pose.p[:2])**2)) < 0.15: + stand_pose = rand_pose( + xlim=stand_x_lim, + ylim=[0, 0.2], + qpos=[0.707, 0.707, 0, 0], + rotate_rand=False, + ) + + self.stand_id = np.random.choice([1, 2], 1)[0] + self.stand = create_actor( + scene=self, + pose=stand_pose, + modelname="078_phonestand", + convex=True, + model_id=self.stand_id, + is_static=True, + ) + self.add_prohibit_area(self.phone, padding=0.15) + self.add_prohibit_area(self.stand, padding=0.15) + + def play_once(self): + # Determine which arm to use based on phone's position (left if phone is on left side, else right) + arm_tag = ArmTag("left" if self.phone.get_pose().p[0] < 0 else "right") + + # Grasp the phone with specified arm + self.move(self.grasp_actor(self.phone, arm_tag=arm_tag, pre_grasp_dis=0.08)) + + # Get stand's functional point as target for placement + stand_func_pose = self.stand.get_functional_point(0) + + # Place the phone onto the stand's functional point with alignment constraint + self.move( + self.place_actor( + self.phone, + arm_tag=arm_tag, + target_pose=stand_func_pose, + functional_point_id=0, + dis=0, + constrain="align", + )) + + self.info["info"] = { + "{A}": f"077_phone/base{self.phone_id}", + "{B}": f"078_phonestand/base{self.stand_id}", + "{a}": str(arm_tag), + } + return self.info + + def check_success(self): + phone_func_pose = np.array(self.phone.get_functional_point(0)) + stand_func_pose = np.array(self.stand.get_functional_point(0)) + eps = np.array([0.045, 0.04, 0.04]) + return (np.all(np.abs(phone_func_pose - stand_func_pose)[:3] < eps) and self.is_left_gripper_open() + and self.is_right_gripper_open()) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_shoe.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_shoe.py new file mode 100644 index 00000000..258c74c6 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/place_shoe.py @@ -0,0 +1,100 @@ +from ._base_task import Base_Task +from .utils import * +import math +import sapien + + +class place_shoe(Base_Task): + + def setup_demo(self, is_test=False, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + self.target_block = create_box( + scene=self, + pose=sapien.Pose([0, -0.08, 0.74], [1, 0, 0, 0]), + half_size=(0.13, 0.05, 0.0005), + color=(0, 0, 1), + is_static=True, + name="box", + ) + self.target_block.config["functional_matrix"] = [[ + [0.0, -1.0, 0.0, 0.0], + [-1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, -1.0, 0], + [0.0, 0.0, 0.0, 1.0], + ], [ + [0.0, -1.0, 0.0, 0.0], + [-1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, -1.0, 0], + [0.0, 0.0, 0.0, 1.0], + ]] + + shoes_pose = rand_pose( + xlim=[-0.25, 0.25], + ylim=[-0.1, 0.05], + ylim_prop=True, + rotate_rand=True, + rotate_lim=[0, 3.14, 0], + qpos=[0.707, 0.707, 0, 0], + ) + while np.sum(pow(shoes_pose.get_p()[:2] - np.zeros(2), 2)) < 0.0225: + shoes_pose = rand_pose( + xlim=[-0.25, 0.25], + ylim=[-0.1, 0.05], + ylim_prop=True, + rotate_rand=True, + rotate_lim=[0, 3.14, 0], + qpos=[0.707, 0.707, 0, 0], + ) + self.shoe_id = np.random.choice([i for i in range(10)]) + self.shoe = create_actor( + scene=self, + pose=shoes_pose, + modelname="041_shoe", + convex=True, + model_id=self.shoe_id, + ) + + self.prohibited_area.append([-0.2, -0.15, 0.2, -0.01]) + self.add_prohibit_area(self.shoe, padding=0.1) + + def play_once(self): + shoe_pose = self.shoe.get_pose().p + arm_tag = ArmTag("left" if shoe_pose[0] < 0 else "right") + + # Grasp the shoe with specified pre-grasp distance and gripper position + self.move(self.grasp_actor(self.shoe, arm_tag=arm_tag, pre_grasp_dis=0.1, gripper_pos=0)) + + # Lift the shoe up by 0.07 meters in z-direction + self.move(self.move_by_displacement(arm_tag=arm_tag, z=0.07)) + + # Get target's functional point as target pose + target_pose = self.target_block.get_functional_point(0) + # Place the shoe on the target with alignment constraint and specified pre-placement distance + self.move( + self.place_actor( + self.shoe, + arm_tag=arm_tag, + target_pose=target_pose, + functional_point_id=0, + pre_dis=0.12, + constrain="align", + )) + # Open the gripper to release the shoe + self.move(self.open_gripper(arm_tag=arm_tag)) + + self.info["info"] = {"{A}": f"041_shoe/base{self.shoe_id}", "{a}": str(arm_tag)} + return self.info + + def check_success(self): + shoe_pose_p = np.array(self.shoe.get_pose().p) + shoe_pose_q = np.array(self.shoe.get_pose().q) + if shoe_pose_q[0] < 0: + shoe_pose_q *= -1 + target_pose_p = np.array([0, -0.08]) + target_pose_q = np.array([0.5, 0.5, -0.5, -0.5]) + eps = np.array([0.05, 0.02, 0.07, 0.07, 0.07, 0.07]) + return (np.all(abs(shoe_pose_p[:2] - target_pose_p) < eps[:2]) + and np.all(abs(shoe_pose_q - target_pose_q) < eps[-4:]) and self.is_left_gripper_open() + and self.is_right_gripper_open()) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/press_stapler.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/press_stapler.py new file mode 100644 index 00000000..d21af598 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/press_stapler.py @@ -0,0 +1,55 @@ +from ._base_task import Base_Task +from .utils import * +from ._GLOBAL_CONFIGS import * + + +class press_stapler(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + rand_pos = rand_pose( + xlim=[-0.2, 0.2], + ylim=[-0.1, 0.05], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, np.pi, 0], + ) + + self.stapler_id = np.random.choice([0, 1, 2, 3, 4, 5, 6], 1)[0] + self.stapler = create_actor(self, + pose=rand_pos, + modelname="048_stapler", + convex=True, + model_id=self.stapler_id, + is_static=True) + + self.add_prohibit_area(self.stapler, padding=0.05) + + def play_once(self): + # Determine which arm to use based on stapler's position (left if negative x, right otherwise) + arm_tag = ArmTag("left" if self.stapler.get_pose().p[0] < 0 else "right") + + # Move arm to the overhead position of the stapler and close the gripper + self.move(self.grasp_actor(self.stapler, arm_tag=arm_tag, pre_grasp_dis=0.1, grasp_dis=0.1, contact_point_id=2)) + self.move(self.close_gripper(arm_tag=arm_tag)) + + # Move the stapler down slightly to press it + self.move( + self.grasp_actor(self.stapler, arm_tag=arm_tag, pre_grasp_dis=0.02, grasp_dis=0.02, contact_point_id=2)) + + self.info["info"] = {"{A}": f"048_stapler/base{self.stapler_id}", "{a}": str(arm_tag)} + return self.info + + def check_success(self): + if self.stage_success_tag: + return True + stapler_pose = self.stapler.get_contact_point(2)[:3] + positions = self.get_gripper_actor_contact_position("048_stapler") + eps = [0.03, 0.03] + for position in positions: + if (np.all(np.abs(position[:2] - stapler_pose[:2]) < eps) and abs(position[2] - stapler_pose[2]) < 0.03): + self.stage_success_tag = True + return True + return False diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/put_bottles_dustbin.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/put_bottles_dustbin.py new file mode 100644 index 00000000..bb26da0c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/put_bottles_dustbin.py @@ -0,0 +1,153 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +from copy import deepcopy + + +class put_bottles_dustbin(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(table_xy_bias=[0.3, 0], **kwags) + + def load_actors(self): + pose_lst = [] + + def create_bottle(model_id): + bottle_pose = rand_pose( + xlim=[-0.25, 0.3], + ylim=[0.03, 0.23], + rotate_rand=False, + rotate_lim=[0, 1, 0], + qpos=[0.707, 0.707, 0, 0], + ) + tag = True + gen_lim = 100 + i = 1 + while tag and i < gen_lim: + tag = False + if np.abs(bottle_pose.p[0]) < 0.05: + tag = True + for pose in pose_lst: + if (np.sum(np.power(np.array(pose[:2]) - np.array(bottle_pose.p[:2]), 2)) < 0.0169): + tag = True + break + if tag: + i += 1 + bottle_pose = rand_pose( + xlim=[-0.25, 0.3], + ylim=[0.03, 0.23], + rotate_rand=False, + rotate_lim=[0, 1, 0], + qpos=[0.707, 0.707, 0, 0], + ) + pose_lst.append(bottle_pose.p[:2]) + bottle = create_actor( + self, + bottle_pose, + modelname="114_bottle", + convex=True, + model_id=model_id, + ) + + return bottle + + self.bottles = [] + self.bottles_data = [] + self.bottle_id = [1, 2, 3] + self.bottle_num = 3 + for i in range(self.bottle_num): + bottle = create_bottle(self.bottle_id[i]) + self.bottles.append(bottle) + self.add_prohibit_area(bottle, padding=0.1) + + self.dustbin = create_actor( + self.scene, + pose=sapien.Pose([-0.45, 0, 0], [0.5, 0.5, 0.5, 0.5]), + modelname="011_dustbin", + convex=True, + is_static=True, + ) + self.delay(2) + self.right_middle_pose = [0, 0.0, 0.88, 0, 1, 0, 0] + + def play_once(self): + # Sort bottles based on their x and y coordinates + bottle_lst = sorted(self.bottles, key=lambda x: [x.get_pose().p[0] > 0, x.get_pose().p[1]]) + + for i in range(self.bottle_num): + bottle = bottle_lst[i] + # Determine which arm to use based on bottle's x position + arm_tag = ArmTag("left" if bottle.get_pose().p[0] < 0 else "right") + + delta_dis = 0.06 + + # Define end position for left arm + left_end_action = Action("left", "move", [-0.35, -0.1, 0.93, 0.65, -0.25, 0.25, 0.65]) + + if arm_tag == "left": + # Grasp the bottle with left arm + self.move(self.grasp_actor(bottle, arm_tag=arm_tag, pre_grasp_dis=0.1)) + # Move left arm up + self.move(self.move_by_displacement(arm_tag, z=0.1)) + # Move left arm to end position + self.move((ArmTag("left"), [left_end_action])) + else: + # Grasp the bottle with right arm while moving left arm to origin + right_action = self.grasp_actor(bottle, arm_tag=arm_tag, pre_grasp_dis=0.1) + right_action[1][0].target_pose[2] += delta_dis + right_action[1][1].target_pose[2] += delta_dis + self.move(right_action, self.back_to_origin("left")) + # Move right arm up + self.move(self.move_by_displacement(arm_tag, z=0.1)) + # Place the bottle at middle position with right arm + self.move( + self.place_actor( + bottle, + target_pose=self.right_middle_pose, + arm_tag=arm_tag, + functional_point_id=0, + pre_dis=0.0, + dis=0.0, + is_open=False, + constrain="align", + )) + # Grasp the bottle with left arm (adjusted height) + left_action = self.grasp_actor(bottle, arm_tag="left", pre_grasp_dis=0.1) + left_action[1][0].target_pose[2] -= delta_dis + left_action[1][1].target_pose[2] -= delta_dis + self.move(left_action) + # Open right gripper + self.move(self.open_gripper(ArmTag("right"))) + # Move left arm to end position while moving right arm to origin + self.move((ArmTag("left"), [left_end_action]), self.back_to_origin("right")) + # Open left gripper + self.move(self.open_gripper("left")) + + self.info["info"] = { + "{A}": f"114_bottle/base{self.bottle_id[0]}", + "{B}": f"114_bottle/base{self.bottle_id[1]}", + "{C}": f"114_bottle/base{self.bottle_id[2]}", + "{D}": f"011_dustbin/base0", + } + return self.info + + def stage_reward(self): + taget_pose = [-0.45, 0] + eps = np.array([0.221, 0.325]) + reward = 0 + reward_step = 1 / 3 + for i in range(self.bottle_num): + bottle_pose = self.bottles[i].get_pose().p + if (np.all(np.abs(bottle_pose[:2] - taget_pose) < eps) and bottle_pose[2] > 0.2 and bottle_pose[2] < 0.7): + reward += reward_step + return reward + + def check_success(self): + taget_pose = [-0.45, 0] + eps = np.array([0.221, 0.325]) + for i in range(self.bottle_num): + bottle_pose = self.bottles[i].get_pose().p + if (np.all(np.abs(bottle_pose[:2] - taget_pose) < eps) and bottle_pose[2] > 0.2 and bottle_pose[2] < 0.7): + continue + return False + return True diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/put_object_cabinet.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/put_object_cabinet.py new file mode 100644 index 00000000..c8e5c10e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/put_object_cabinet.py @@ -0,0 +1,123 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import glob + + +class put_object_cabinet(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags, table_static=False) + + def load_actors(self): + self.model_name = "036_cabinet" + self.model_id = 46653 + self.cabinet = rand_create_sapien_urdf_obj( + scene=self, + modelname=self.model_name, + modelid=self.model_id, + xlim=[-0.05, 0.05], + ylim=[0.155, 0.155], + rotate_rand=False, + rotate_lim=[0, 0, np.pi / 16], + qpos=[1, 0, 0, 1], + fix_root_link=True, + ) + rand_pos = rand_pose( + xlim=[-0.25, 0.25], + ylim=[-0.2, -0.1], + qpos=[0.707, 0.707, 0.0, 0.0], + rotate_rand=True, + rotate_lim=[0, np.pi / 3, 0], + ) + while abs(rand_pos.p[0]) < 0.2: + rand_pos = rand_pose( + xlim=[-0.32, 0.32], + ylim=[-0.2, -0.1], + qpos=[0.707, 0.707, 0.0, 0.0], + rotate_rand=True, + rotate_lim=[0, np.pi / 3, 0], + ) + + def get_available_model_ids(modelname): + asset_path = os.path.join("assets/objects", modelname) + json_files = glob.glob(os.path.join(asset_path, "model_data*.json")) + available_ids = [] + for file in json_files: + base = os.path.basename(file) + try: + idx = int(base.replace("model_data", "").replace(".json", "")) + available_ids.append(idx) + except ValueError: + continue + return available_ids + + object_list = [ + "047_mouse", + "048_stapler", + "057_toycar", + "073_rubikscube", + "075_bread", + "077_phone", + "081_playingcards", + "112_tea-box", + "113_coffee-box", + "107_soap", + ] + self.selected_modelname = np.random.choice(object_list) + available_model_ids = get_available_model_ids(self.selected_modelname) + if not available_model_ids: + raise ValueError(f"No available model_data.json files found for {self.selected_modelname}") + self.selected_model_id = np.random.choice(available_model_ids) + self.object = create_actor( + scene=self, + pose=rand_pos, + modelname=self.selected_modelname, + convex=True, + model_id=self.selected_model_id, + ) + self.object.set_mass(0.01) + self.add_prohibit_area(self.object, padding=0.01) + self.add_prohibit_area(self.cabinet, padding=0.01) + self.prohibited_area.append([-0.15, -0.3, 0.15, 0.3]) + + def play_once(self): + arm_tag = ArmTag("right" if self.object.get_pose().p[0] > 0 else "left") + self.arm_tag = arm_tag + self.origin_z = self.object.get_pose().p[2] + + # Grasp the object and grasp the drawer bar + self.move(self.grasp_actor(self.object, arm_tag=arm_tag, pre_grasp_dis=0.1)) + self.move(self.grasp_actor(self.cabinet, arm_tag=arm_tag.opposite, pre_grasp_dis=0.05)) + + # Pull the drawer + for _ in range(4): + self.move(self.move_by_displacement(arm_tag=arm_tag.opposite, y=-0.04)) + + # Lift the object + self.move(self.move_by_displacement(arm_tag=arm_tag, z=0.15)) + + # Place the object into the cabinet + target_pose = self.cabinet.get_functional_point(0) + self.move(self.place_actor( + self.object, + arm_tag=arm_tag, + target_pose=target_pose, + pre_dis=0.13, + dis=0.1, + )) + + self.info["info"] = { + "{A}": f"{self.selected_modelname}/base{self.selected_model_id}", + "{B}": f"036_cabinet/base{0}", + "{a}": str(arm_tag), + "{b}": str(arm_tag.opposite), + } + return self.info + + def check_success(self): + object_pose = self.object.get_pose().p + target_pose = self.cabinet.get_functional_point(0) + tag = np.all(abs(object_pose[:2] - target_pose[:2]) < np.array([0.05, 0.05])) + return ((object_pose[2] - self.origin_z) > 0.007 and (object_pose[2] - self.origin_z) < 0.12 and tag + and (self.robot.is_left_gripper_open() if self.arm_tag == "left" else self.robot.is_right_gripper_open())) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/robot/__init__.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/robot/__init__.py new file mode 100644 index 00000000..67b88137 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/robot/__init__.py @@ -0,0 +1,2 @@ +from .robot import * +from .planner import * diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/robot/ik.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/robot/ik.py new file mode 100644 index 00000000..46409041 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/robot/ik.py @@ -0,0 +1 @@ +# TODO diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/robot/planner.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/robot/planner.py new file mode 100644 index 00000000..21ae4326 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/robot/planner.py @@ -0,0 +1,434 @@ +import mplib.planner +import mplib +import numpy as np +import pdb +import traceback +import numpy as np +import toppra as ta +from mplib.sapien_utils import SapienPlanner, SapienPlanningWorld +import transforms3d as t3d +import envs._GLOBAL_CONFIGS as CONFIGS + + +try: + # ********************** CuroboPlanner (optional) ********************** + from curobo.types.math import Pose as CuroboPose + import time + from curobo.types.robot import JointState + from curobo.wrap.reacher.motion_gen import ( + MotionGen, + MotionGenConfig, + MotionGenPlanConfig, + PoseCostMetric, + ) + from curobo.util import logger + import torch + import yaml + from curobo.util import logger + logger.setup_logger(level="error", logger_name="curobo") + + class CuroboPlanner: + + def __init__( + self, + robot_origion_pose, + active_joints_name, + all_joints, + yml_path=None, + ): + super().__init__() + ta.setup_logging("CRITICAL") # hide logging + logger.setup_logger(level="error", logger_name="'curobo") + + if yml_path != None: + self.yml_path = yml_path + else: + raise ValueError("[Planner.py]: CuroboPlanner yml_path is None!") + self.robot_origion_pose = robot_origion_pose + self.active_joints_name = active_joints_name + self.all_joints = all_joints + + # translate from baselink to arm's base + with open(self.yml_path, "r") as f: + yml_data = yaml.safe_load(f) + self.frame_bias = yml_data["planner"]["frame_bias"] + + # motion generation + if True: + world_config = { + "cuboid": { + "table": { + "dims": [0.7, 2, 0.04], # x, y, z + "pose": [ + self.robot_origion_pose.p[1], + 0.0, + 0.74 - self.robot_origion_pose.p[2], + 1, + 0, + 0, + 0.0, + ], # x, y, z, qw, qx, qy, qz + }, + } + } + motion_gen_config = MotionGenConfig.load_from_robot_config( + self.yml_path, + world_config, + interpolation_dt=1 / 250, + num_trajopt_seeds=1, + ) + + self.motion_gen = MotionGen(motion_gen_config) + self.motion_gen.warmup() + motion_gen_config = MotionGenConfig.load_from_robot_config( + self.yml_path, + world_config, + interpolation_dt=1 / 250, + num_trajopt_seeds=1, + num_graph_seeds=1, + ) + self.motion_gen_batch = MotionGen(motion_gen_config) + self.motion_gen_batch.warmup(batch=CONFIGS.ROTATE_NUM) + + def plan_path( + self, + curr_joint_pos, + target_gripper_pose, + constraint_pose=None, + arms_tag=None, + ): + world_base_pose = np.concatenate([ + np.array(self.robot_origion_pose.p), + np.array(self.robot_origion_pose.q), + ]) + world_target_pose = np.concatenate([np.array(target_gripper_pose.p), np.array(target_gripper_pose.q)]) + target_pose_p, target_pose_q = self._trans_from_world_to_base(world_base_pose, world_target_pose) + if not ("aloha-agilex" in self.yml_path): + target_pose_p[0] += self.frame_bias[0] + target_pose_p[1] += self.frame_bias[1] + target_pose_p[2] += self.frame_bias[2] + else: # patch for aloha-agilex + T_target = t3d.affines.compose(target_pose_p, t3d.quaternions.quat2mat(target_pose_q), [1, 1, 1]) + T_bias = t3d.affines.compose(self.frame_bias, np.eye(3), [1, 1, 1]) + + if arms_tag == "left": + rot = t3d.axangles.axangle2mat([0, 0, 1], -0.02) + elif arms_tag == "right": + rot = t3d.axangles.axangle2mat([0, 0, 1], -0.01) + else: + raise ValueError(f"Invalid arms_tag: {arms_tag}") + + T_rot = t3d.affines.compose([0, 0, 0], rot, [1, 1, 1]) + T_new = T_rot @ T_bias @ T_target + target_pose_p = T_new[:3, 3] + target_pose_q = t3d.quaternions.mat2quat(T_new[:3, :3]) + + goal_pose_of_ee = CuroboPose.from_list(list(target_pose_p) + list(target_pose_q)) + joint_indices = [self.all_joints.index(name) for name in self.active_joints_name if name in self.all_joints] + joint_angles = [curr_joint_pos[index] for index in joint_indices] + joint_angles = [round(angle, 5) for angle in joint_angles] # avoid the precision problem + start_joint_states = JointState.from_position( + torch.tensor(joint_angles).cuda().reshape(1, -1), + joint_names=self.active_joints_name, + ) + # plan + plan_config = MotionGenPlanConfig(max_attempts=10) + if constraint_pose is not None: + pose_cost_metric = PoseCostMetric( + hold_partial_pose=True, + hold_vec_weight=self.motion_gen.tensor_args.to_device(constraint_pose), + ) + plan_config.pose_cost_metric = pose_cost_metric + + result = self.motion_gen.plan_single(start_joint_states, goal_pose_of_ee, plan_config) + + # output + res_result = dict() + if result.success.item() == False: + res_result["status"] = "Fail" + return res_result + else: + res_result["status"] = "Success" + res_result["position"] = np.array(result.interpolated_plan.position.to("cpu")) + res_result["velocity"] = np.array(result.interpolated_plan.velocity.to("cpu")) + return res_result + + def plan_batch( + self, + curr_joint_pos, + target_gripper_pose_list, + constraint_pose=None, + arms_tag=None, + ): + """ + Plan a batch of trajectories for multiple target poses. + + Input: + - curr_joint_pos: List of current joint angles (1 x n) + - target_gripper_pose_list: List of target poses [sapien.Pose, sapien.Pose, ...] + + Output: + - result['status']: numpy array of string values indicating "Success"/"Fail" for each pose + - result['position']: numpy array of joint positions with shape (n x m x l) + where n is number of target poses, m is number of waypoints, l is number of joints + - result['velocity']: numpy array of joint velocities with same shape as position + """ + + num_poses = len(target_gripper_pose_list) + # transformation from world to arm's base + world_base_pose = np.concatenate([ + np.array(self.robot_origion_pose.p), + np.array(self.robot_origion_pose.q), + ]) + poses_list = [] + for target_gripper_pose in target_gripper_pose_list: + world_target_pose = np.concatenate([np.array(target_gripper_pose.p), np.array(target_gripper_pose.q)]) + base_target_pose_p, base_target_pose_q = self._trans_from_world_to_base(world_base_pose, world_target_pose) + + if not ("aloha-agilex" in self.yml_path): + base_target_pose_p[0] += self.frame_bias[0] + base_target_pose_p[1] += self.frame_bias[1] + base_target_pose_p[2] += self.frame_bias[2] + else: # patch for aloha-agilex + T_target = t3d.affines.compose(base_target_pose_p, t3d.quaternions.quat2mat(base_target_pose_q), [1, 1, 1]) + T_bias = t3d.affines.compose(self.frame_bias, np.eye(3), [1, 1, 1]) + + if arms_tag == "left": + rot = t3d.axangles.axangle2mat([0, 0, 1], -0.02) + elif arms_tag == "right": + rot = t3d.axangles.axangle2mat([0, 0, 1], -0.01) + else: + raise ValueError(f"Invalid arms_tag: {arms_tag}") + + T_rot = t3d.affines.compose([0, 0, 0], rot, [1, 1, 1]) + T_new = T_rot @ T_bias @ T_target + base_target_pose_p = T_new[:3, 3] + base_target_pose_q = t3d.quaternions.mat2quat(T_new[:3, :3]) + + base_target_pose_list = list(base_target_pose_p) + list(base_target_pose_q) + poses_list.append(base_target_pose_list) + + poses_cuda = torch.tensor(poses_list, dtype=torch.float32).cuda() + goal_pose_of_ee = CuroboPose(poses_cuda[:, :3], poses_cuda[:, 3:]) + joint_indices = [self.all_joints.index(name) for name in self.active_joints_name if name in self.all_joints] + joint_angles = [curr_joint_pos[index] for index in joint_indices] + joint_angles = [round(angle, 5) for angle in joint_angles] # avoid the precision problem + joint_angles_cuda = (torch.tensor(joint_angles, dtype=torch.float32).cuda().reshape(1, -1)) + joint_angles_cuda = torch.cat([joint_angles_cuda] * num_poses, dim=0) + start_joint_states = JointState.from_position(joint_angles_cuda, joint_names=self.active_joints_name) + # plan + plan_config = MotionGenPlanConfig(max_attempts=10) + if constraint_pose is not None: + pose_cost_metric = PoseCostMetric( + hold_partial_pose=True, + hold_vec_weight=self.motion_gen.tensor_args.to_device(constraint_pose), + ) + plan_config.pose_cost_metric = pose_cost_metric + + try: + result = self.motion_gen_batch.plan_batch(start_joint_states, goal_pose_of_ee, plan_config) + except Exception as e: + return {"status": ["Failure" for i in range(10)]} + + # output + res_result = dict() + # Convert boolean success values to "Success"/"Failure" strings + success_array = result.success.cpu().numpy() + status_array = np.array(["Success" if s else "Failure" for s in success_array], dtype=object) + res_result["status"] = status_array + + if np.all(res_result["status"] == "Failure"): + return res_result + + res_result["position"] = np.array(result.interpolated_plan.position.to("cpu")) + res_result["velocity"] = np.array(result.interpolated_plan.velocity.to("cpu")) + return res_result + + def plan_grippers(self, now_val, target_val): + num_step = 200 + dis_val = target_val - now_val + step = dis_val / num_step + res = {} + vals = np.linspace(now_val, target_val, num_step) + res["num_step"] = num_step + res["per_step"] = step + res["result"] = vals + return res + + def _trans_from_world_to_base(self, base_pose, target_pose): + ''' + transform target pose from world frame to base frame + base_pose: np.array([x, y, z, qw, qx, qy, qz]) + target_pose: np.array([x, y, z, qw, qx, qy, qz]) + ''' + base_p, base_q = base_pose[0:3], base_pose[3:] + target_p, target_q = target_pose[0:3], target_pose[3:] + rel_p = target_p - base_p + wRb = t3d.quaternions.quat2mat(base_q) + wRt = t3d.quaternions.quat2mat(target_q) + result_p = wRb.T @ rel_p + result_q = t3d.quaternions.mat2quat(wRb.T @ wRt) + return result_p, result_q + +except Exception as e: + print('[planner.py]: Something wrong happened when importing CuroboPlanner! Please check if Curobo is installed correctly. If the problem still exists, you can install Curobo from https://github.com/NVlabs/curobo manually.') + print('Exception traceback:') + traceback.print_exc() + + +# ********************** MplibPlanner ********************** +class MplibPlanner: + # links=None, joints=None + def __init__( + self, + urdf_path, + srdf_path, + move_group, + robot_origion_pose, + robot_entity, + planner_type="mplib_RRT", + scene=None, + ): + super().__init__() + ta.setup_logging("CRITICAL") # hide logging + + links = [link.get_name() for link in robot_entity.get_links()] + joints = [joint.get_name() for joint in robot_entity.get_active_joints()] + + if scene is None: + self.planner = mplib.Planner( + urdf=urdf_path, + srdf=srdf_path, + move_group=move_group, + user_link_names=links, + user_joint_names=joints, + use_convex=False, + ) + self.planner.set_base_pose(robot_origion_pose) + else: + planning_world = SapienPlanningWorld(scene, [robot_entity]) + self.planner = SapienPlanner(planning_world, move_group) + + self.planner_type = planner_type + self.plan_step_lim = 2500 + self.TOPP = self.planner.TOPP + + def show_info(self): + print("joint_limits", self.planner.joint_limits) + print("joint_acc_limits", self.planner.joint_acc_limits) + + def plan_pose( + self, + now_qpos, + target_pose, + use_point_cloud=False, + use_attach=False, + arms_tag=None, + try_times=2, + log=True, + ): + result = {} + result["status"] = "Fail" + + now_try_times = 1 + while result["status"] != "Success" and now_try_times < try_times: + result = self.planner.plan_pose( + goal_pose=target_pose, + current_qpos=np.array(now_qpos), + time_step=1 / 250, + planning_time=5, + # rrt_range=0.05 + # =================== mplib 0.1.1 =================== + # use_point_cloud=use_point_cloud, + # use_attach=use_attach, + # planner_name="RRTConnect" + ) + now_try_times += 1 + + if result["status"] != "Success": + if log: + print(f"\n {arms_tag} arm planning failed ({result['status']}) !") + else: + n_step = result["position"].shape[0] + if n_step > self.plan_step_lim: + if log: + print(f"\n {arms_tag} arm planning wrong! (step = {n_step})") + result["status"] = "Fail" + + return result + + def plan_screw( + self, + now_qpos, + target_pose, + use_point_cloud=False, + use_attach=False, + arms_tag=None, + log=False, + ): + """ + Interpolative planning with screw motion. + Will not avoid collision and will fail if the path contains collision. + """ + result = self.planner.plan_screw( + goal_pose=target_pose, + current_qpos=now_qpos, + time_step=1 / 250, + # =================== mplib 0.1.1 =================== + # use_point_cloud=use_point_cloud, + # use_attach=use_attach, + ) + + # plan fail + if result["status"] != "Success": + if log: + print(f"\n {arms_tag} arm planning failed ({result['status']}) !") + # return result + else: + n_step = result["position"].shape[0] + # plan step lim + if n_step > self.plan_step_lim: + if log: + print(f"\n {arms_tag} arm planning wrong! (step = {n_step})") + result["status"] = "Fail" + + return result + + def plan_path( + self, + now_qpos, + target_pose, + use_point_cloud=False, + use_attach=False, + arms_tag=None, + log=True, + ): + """ + Interpolative planning with screw motion. + Will not avoid collision and will fail if the path contains collision. + """ + if self.planner_type == "mplib_RRT": + result = self.plan_pose( + now_qpos, + target_pose, + use_point_cloud, + use_attach, + arms_tag, + try_times=10, + log=log, + ) + elif self.planner_type == "mplib_screw": + result = self.plan_screw(now_qpos, target_pose, use_point_cloud, use_attach, arms_tag, log) + + return result + + def plan_grippers(self, now_val, target_val): + num_step = 200 # TODO + dis_val = target_val - now_val + per_step = dis_val / num_step + res = {} + vals = np.linspace(now_val, target_val, num_step) + res["num_step"] = num_step + res["per_step"] = per_step # dis per step + res["result"] = vals + return res diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/robot/robot.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/robot/robot.py new file mode 100644 index 00000000..9dfc08f5 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/robot/robot.py @@ -0,0 +1,714 @@ +import sapien.core as sapien +import numpy as np +import pdb +from .planner import MplibPlanner +import numpy as np +import toppra as ta +import math +import yaml +import os +import transforms3d as t3d +from copy import deepcopy +import sapien.core as sapien +import envs._GLOBAL_CONFIGS as CONFIGS +from envs.utils import transforms +from .planner import CuroboPlanner +import torch.multiprocessing as mp + + +class Robot: + + def __init__(self, scene, need_topp=False, **kwargs): + super().__init__() + ta.setup_logging("CRITICAL") # hide logging + self._init_robot_(scene, need_topp, **kwargs) + + def _init_robot_(self, scene, need_topp=False, **kwargs): + # self.dual_arm = dual_arm_tag + # self.plan_success = True + + self.left_js = None + self.right_js = None + + left_embodiment_args = kwargs["left_embodiment_config"] + right_embodiment_args = kwargs["right_embodiment_config"] + left_robot_file = kwargs["left_robot_file"] + right_robot_file = kwargs["right_robot_file"] + + self.need_topp = need_topp + + self.left_urdf_path = os.path.join(left_robot_file, left_embodiment_args["urdf_path"]) + self.left_srdf_path = left_embodiment_args.get("srdf_path", None) + self.left_curobo_yml_path = os.path.join(left_robot_file, "curobo.yml") + if self.left_srdf_path is not None: + self.left_srdf_path = os.path.join(left_robot_file, self.left_srdf_path) + self.left_joint_stiffness = left_embodiment_args.get("joint_stiffness", 1000) + self.left_joint_damping = left_embodiment_args.get("joint_damping", 200) + self.left_gripper_stiffness = left_embodiment_args.get("gripper_stiffness", 1000) + self.left_gripper_damping = left_embodiment_args.get("gripper_damping", 200) + self.left_planner_type = left_embodiment_args.get("planner", "mplib_RRT") + self.left_move_group = left_embodiment_args["move_group"][0] + self.left_ee_name = left_embodiment_args["ee_joints"][0] + self.left_arm_joints_name = left_embodiment_args["arm_joints_name"][0] + self.left_gripper_name = left_embodiment_args["gripper_name"][0] + self.left_gripper_bias = left_embodiment_args["gripper_bias"] + self.left_gripper_scale = left_embodiment_args["gripper_scale"] + self.left_homestate = left_embodiment_args.get("homestate", [[0] * len(self.left_arm_joints_name)])[0] + self.left_fix_gripper_name = left_embodiment_args.get("fix_gripper_name", []) + self.left_delta_matrix = np.array(left_embodiment_args.get("delta_matrix", [[1, 0, 0], [0, 1, 0], [0, 0, 1]])) + self.left_inv_delta_matrix = np.linalg.inv(self.left_delta_matrix) + self.left_global_trans_matrix = np.array( + left_embodiment_args.get("global_trans_matrix", [[1, 0, 0], [0, 1, 0], [0, 0, 1]])) + + _entity_origion_pose = left_embodiment_args.get("robot_pose", [[0, -0.65, 0, 1, 0, 0, 1]])[0] + _entity_origion_pose = sapien.Pose(_entity_origion_pose[:3], _entity_origion_pose[-4:]) + self.left_entity_origion_pose = deepcopy(_entity_origion_pose) + + self.right_urdf_path = os.path.join(right_robot_file, right_embodiment_args["urdf_path"]) + self.right_srdf_path = right_embodiment_args.get("srdf_path", None) + if self.right_srdf_path is not None: + self.right_srdf_path = os.path.join(right_robot_file, self.right_srdf_path) + self.right_curobo_yml_path = os.path.join(right_robot_file, "curobo.yml") + self.right_joint_stiffness = right_embodiment_args.get("joint_stiffness", 1000) + self.right_joint_damping = right_embodiment_args.get("joint_damping", 200) + self.right_gripper_stiffness = right_embodiment_args.get("gripper_stiffness", 1000) + self.right_gripper_damping = right_embodiment_args.get("gripper_damping", 200) + self.right_planner_type = right_embodiment_args.get("planner", "mplib_RRT") + self.right_move_group = right_embodiment_args["move_group"][1] + self.right_ee_name = right_embodiment_args["ee_joints"][1] + self.right_arm_joints_name = right_embodiment_args["arm_joints_name"][1] + self.right_gripper_name = right_embodiment_args["gripper_name"][1] + self.right_gripper_bias = right_embodiment_args["gripper_bias"] + self.right_gripper_scale = right_embodiment_args["gripper_scale"] + self.right_homestate = right_embodiment_args.get("homestate", [[1] * len(self.right_arm_joints_name)])[1] + self.right_fix_gripper_name = right_embodiment_args.get("fix_gripper_name", []) + self.right_delta_matrix = np.array(right_embodiment_args.get("delta_matrix", [[1, 0, 0], [0, 1, 0], [0, 0, 1]])) + self.right_inv_delta_matrix = np.linalg.inv(self.right_delta_matrix) + self.right_global_trans_matrix = np.array( + right_embodiment_args.get("global_trans_matrix", [[1, 0, 0], [0, 1, 0], [0, 0, 1]])) + + _entity_origion_pose = right_embodiment_args.get("robot_pose", [[0, -0.65, 0, 1, 0, 0, 1]]) + _entity_origion_pose = _entity_origion_pose[0 if len(_entity_origion_pose) == 1 else 1] + _entity_origion_pose = sapien.Pose(_entity_origion_pose[:3], _entity_origion_pose[-4:]) + self.right_entity_origion_pose = deepcopy(_entity_origion_pose) + self.is_dual_arm = kwargs["dual_arm_embodied"] + + self.left_rotate_lim = left_embodiment_args.get("rotate_lim", [0, 0]) + self.right_rotate_lim = right_embodiment_args.get("rotate_lim", [0, 0]) + + self.left_perfect_direction = left_embodiment_args.get("grasp_perfect_direction", + ["front_right", "front_left"])[0] + self.right_perfect_direction = right_embodiment_args.get("grasp_perfect_direction", + ["front_right", "front_left"])[1] + + if self.is_dual_arm: + loader: sapien.URDFLoader = scene.create_urdf_loader() + loader.fix_root_link = True + self._entity = loader.load(self.left_urdf_path) + self.left_entity = self._entity + self.right_entity = self._entity + else: + arms_dis = kwargs["embodiment_dis"] + self.left_entity_origion_pose.p += [-arms_dis / 2, 0, 0] + self.right_entity_origion_pose.p += [arms_dis / 2, 0, 0] + left_loader: sapien.URDFLoader = scene.create_urdf_loader() + left_loader.fix_root_link = True + right_loader: sapien.URDFLoader = scene.create_urdf_loader() + right_loader.fix_root_link = True + self.left_entity = left_loader.load(self.left_urdf_path) + self.right_entity = right_loader.load(self.right_urdf_path) + + self.left_entity.set_root_pose(self.left_entity_origion_pose) + self.right_entity.set_root_pose(self.right_entity_origion_pose) + + def reset(self, scene, need_topp=False, **kwargs): + self._init_robot_(scene, need_topp, **kwargs) + + if self.communication_flag: + if hasattr(self, "left_conn") and self.left_conn: + self.left_conn.send({"cmd": "reset"}) + _ = self.left_conn.recv() + if hasattr(self, "right_conn") and self.right_conn: + self.right_conn.send({"cmd": "reset"}) + _ = self.right_conn.recv() + else: + if not isinstance(self.left_planner, CuroboPlanner) or not isinstance(self.right_planner, CuroboPlanner): + self.set_planner(scene=scene) + + self.init_joints() + + def get_grasp_perfect_direction(self, arm_tag): + if arm_tag == "left": + return self.left_perfect_direction + elif arm_tag == "right": + return self.right_perfect_direction + + def create_target_pose_list(self, origin_pose, center_pose, arm_tag=None): + res_lst = [] + rotate_lim = (self.left_rotate_lim if arm_tag == "left" else self.right_rotate_lim) + rotate_step = (rotate_lim[1] - rotate_lim[0]) / CONFIGS.ROTATE_NUM + for i in range(CONFIGS.ROTATE_NUM): + now_pose = transforms.rotate_along_axis( + origin_pose, + center_pose, + [0, 1, 0], + rotate_step * i + rotate_lim[0], + axis_type="target", + towards=[0, -1, 0], + ) + res_lst.append(now_pose) + return res_lst + + def get_constraint_pose(self, ori_vec: list, arm_tag=None): + inv_delta_matrix = (self.left_inv_delta_matrix if arm_tag == "left" else self.right_inv_delta_matrix) + return ori_vec[:3] + (ori_vec[-3:] @ np.linalg.inv(inv_delta_matrix)).tolist() + + def init_joints(self): + if self.left_entity is None or self.right_entity is None: + raise ValueError("Robote entity is None") + + self.left_active_joints = self.left_entity.get_active_joints() + self.right_active_joints = self.right_entity.get_active_joints() + + self.left_ee = self.left_entity.find_joint_by_name(self.left_ee_name) + self.right_ee = self.right_entity.find_joint_by_name(self.right_ee_name) + + self.left_gripper_val = 0.0 + self.right_gripper_val = 0.0 + + self.left_arm_joints = [self.left_entity.find_joint_by_name(i) for i in self.left_arm_joints_name] + self.right_arm_joints = [self.right_entity.find_joint_by_name(i) for i in self.right_arm_joints_name] + + def get_gripper_joints(find, gripper_name: str): + gripper = [(find(gripper_name["base"]), 1.0, 0.0)] + for g in gripper_name["mimic"]: + gripper.append((find(g[0]), g[1], g[2])) + return gripper + + self.left_gripper = get_gripper_joints(self.left_entity.find_joint_by_name, self.left_gripper_name) + self.right_gripper = get_gripper_joints(self.right_entity.find_joint_by_name, self.right_gripper_name) + self.gripper_name = deepcopy(self.left_fix_gripper_name) + deepcopy(self.right_fix_gripper_name) + + for g in self.left_gripper: + self.gripper_name.append(g[0].child_link.get_name()) + for g in self.right_gripper: + self.gripper_name.append(g[0].child_link.get_name()) + + # camera link id + self.left_camera = self.left_entity.find_link_by_name("left_camera") + if self.left_camera is None: + self.left_camera = self.left_entity.find_link_by_name("camera") + if self.left_camera is None: + print("No left camera link") + self.left_camera = self.left_entity.get_links()[0] + + self.right_camera = self.right_entity.find_link_by_name("right_camera") + if self.right_camera is None: + self.right_camera = self.right_entity.find_link_by_name("camera") + if self.right_camera is None: + print("No right camera link") + self.right_camera = self.right_entity.get_links()[0] + + for i, joint in enumerate(self.left_active_joints): + if joint not in self.left_gripper: + joint.set_drive_property(stiffness=self.left_joint_stiffness, damping=self.left_joint_damping) + for i, joint in enumerate(self.right_active_joints): + if joint not in self.right_gripper: + joint.set_drive_property( + stiffness=self.right_joint_stiffness, + damping=self.right_joint_damping, + ) + + for joint in self.left_gripper: + joint[0].set_drive_property(stiffness=self.left_gripper_stiffness, damping=self.left_gripper_damping) + for joint in self.right_gripper: + joint[0].set_drive_property( + stiffness=self.right_gripper_stiffness, + damping=self.right_gripper_damping, + ) + + def move_to_homestate(self): + for i, joint in enumerate(self.left_arm_joints): + joint.set_drive_target(self.left_homestate[i]) + + for i, joint in enumerate(self.right_arm_joints): + joint.set_drive_target(self.right_homestate[i]) + + def set_origin_endpose(self): + self.left_original_pose = self.get_left_ee_pose() + self.right_original_pose = self.get_right_ee_pose() + + def print_info(self): + print( + "active joints: ", + [joint.get_name() for joint in self.left_active_joints + self.right_active_joints], + ) + print( + "all links: ", + [link.get_name() for link in self.left_entity.get_links() + self.right_entity.get_links()], + ) + print("left arm joints: ", [joint.get_name() for joint in self.left_arm_joints]) + print("right arm joints: ", [joint.get_name() for joint in self.right_arm_joints]) + print("left gripper: ", [joint[0].get_name() for joint in self.left_gripper]) + print("right gripper: ", [joint[0].get_name() for joint in self.right_gripper]) + print("left ee: ", self.left_ee.get_name()) + print("right ee: ", self.right_ee.get_name()) + + def set_planner(self, scene=None): + abs_left_curobo_yml_path = os.path.join(CONFIGS.ROOT_PATH, self.left_curobo_yml_path) + abs_right_curobo_yml_path = os.path.join(CONFIGS.ROOT_PATH, self.right_curobo_yml_path) + + self.communication_flag = (abs_left_curobo_yml_path != abs_right_curobo_yml_path) + + if self.is_dual_arm: + abs_left_curobo_yml_path = abs_left_curobo_yml_path.replace("curobo.yml", "curobo_left.yml") + abs_right_curobo_yml_path = abs_right_curobo_yml_path.replace("curobo.yml", "curobo_right.yml") + + if not self.communication_flag: + self.left_planner = CuroboPlanner(self.left_entity_origion_pose, + self.left_arm_joints_name, + [joint.get_name() for joint in self.left_entity.get_active_joints()], + yml_path=abs_left_curobo_yml_path) + self.right_planner = CuroboPlanner(self.right_entity_origion_pose, + self.right_arm_joints_name, + [joint.get_name() for joint in self.right_entity.get_active_joints()], + yml_path=abs_right_curobo_yml_path) + else: + self.left_conn, left_child_conn = mp.Pipe() + self.right_conn, right_child_conn = mp.Pipe() + + left_args = { + "origin_pose": self.left_entity_origion_pose, + "joints_name": self.left_arm_joints_name, + "all_joints": [joint.get_name() for joint in self.left_entity.get_active_joints()], + "yml_path": abs_left_curobo_yml_path + } + + right_args = { + "origin_pose": self.right_entity_origion_pose, + "joints_name": self.right_arm_joints_name, + "all_joints": [joint.get_name() for joint in self.right_entity.get_active_joints()], + "yml_path": abs_right_curobo_yml_path + } + + self.left_proc = mp.Process(target=planner_process_worker, args=(left_child_conn, left_args)) + self.right_proc = mp.Process(target=planner_process_worker, args=(right_child_conn, right_args)) + + self.left_proc.daemon = True + self.right_proc.daemon = True + + self.left_proc.start() + self.right_proc.start() + + if self.need_topp: + self.left_mplib_planner = MplibPlanner( + self.left_urdf_path, + self.left_srdf_path, + self.left_move_group, + self.left_entity_origion_pose, + self.left_entity, + self.left_planner_type, + scene, + ) + self.right_mplib_planner = MplibPlanner( + self.right_urdf_path, + self.right_srdf_path, + self.right_move_group, + self.right_entity_origion_pose, + self.right_entity, + self.right_planner_type, + scene, + ) + + def update_world_pcd(self, world_pcd): + try: + self.left_planner.update_point_cloud(world_pcd, resolution=0.02) + self.right_planner.update_point_cloud(world_pcd, resolution=0.02) + except: + print("Update world pointcloud wrong!") + + def _trans_from_gripper_to_endlink(self, target_pose, arm_tag=None): + gripper_bias = (self.left_gripper_bias if arm_tag == "left" else self.right_gripper_bias) + inv_delta_matrix = (self.left_inv_delta_matrix if arm_tag == "left" else self.right_inv_delta_matrix) + target_pose_arr = np.array(target_pose) + gripper_pose_pos, gripper_pose_quat = deepcopy(target_pose_arr[0:3]), deepcopy(target_pose_arr[-4:]) + gripper_pose_mat = t3d.quaternions.quat2mat(gripper_pose_quat) + gripper_pose_pos += gripper_pose_mat @ np.array([0.12 - gripper_bias, 0, 0]).T + gripper_pose_mat = gripper_pose_mat @ inv_delta_matrix + gripper_pose_quat = t3d.quaternions.mat2quat(gripper_pose_mat) + return sapien.Pose(gripper_pose_pos, gripper_pose_quat) + + def left_plan_grippers(self, now_val, target_val): + if self.communication_flag: + self.left_conn.send({"cmd": "plan_grippers", "now_val": now_val, "target_val": target_val}) + return self.left_conn.recv() + else: + return self.left_planner.plan_grippers(now_val, target_val) + + def right_plan_grippers(self, now_val, target_val): + if self.communication_flag: + self.right_conn.send({"cmd": "plan_grippers", "now_val": now_val, "target_val": target_val}) + return self.right_conn.recv() + else: + return self.right_planner.plan_grippers(now_val, target_val) + + def left_plan_multi_path( + self, + target_lst, + constraint_pose=None, + use_point_cloud=False, + use_attach=False, + last_qpos=None, + ): + if constraint_pose is not None: + constraint_pose = self.get_constraint_pose(constraint_pose, arm_tag="left") + if last_qpos is None: + now_qpos = self.left_entity.get_qpos() + else: + now_qpos = deepcopy(last_qpos) + target_lst_copy = deepcopy(target_lst) + for i in range(len(target_lst_copy)): + target_lst_copy[i] = self._trans_from_gripper_to_endlink(target_lst_copy[i], arm_tag="left") + + if self.communication_flag: + self.left_conn.send({ + "cmd": "plan_batch", + "qpos": now_qpos, + "target_pose_list": target_lst_copy, + "constraint_pose": constraint_pose, + "arms_tag": "left", + }) + return self.left_conn.recv() + else: + return self.left_planner.plan_batch( + now_qpos, + target_lst_copy, + constraint_pose=constraint_pose, + arms_tag="left", + ) + + def right_plan_multi_path( + self, + target_lst, + constraint_pose=None, + use_point_cloud=False, + use_attach=False, + last_qpos=None, + ): + if constraint_pose is not None: + constraint_pose = self.get_constraint_pose(constraint_pose, arm_tag="right") + if last_qpos is None: + now_qpos = self.right_entity.get_qpos() + else: + now_qpos = deepcopy(last_qpos) + target_lst_copy = deepcopy(target_lst) + for i in range(len(target_lst_copy)): + target_lst_copy[i] = self._trans_from_gripper_to_endlink(target_lst_copy[i], arm_tag="right") + + if self.communication_flag: + self.right_conn.send({ + "cmd": "plan_batch", + "qpos": now_qpos, + "target_pose_list": target_lst_copy, + "constraint_pose": constraint_pose, + "arms_tag": "right", + }) + return self.right_conn.recv() + else: + return self.right_planner.plan_batch( + now_qpos, + target_lst_copy, + constraint_pose=constraint_pose, + arms_tag="right", + ) + + def left_plan_path( + self, + target_pose, + constraint_pose=None, + use_point_cloud=False, + use_attach=False, + last_qpos=None, + ): + if constraint_pose is not None: + constraint_pose = self.get_constraint_pose(constraint_pose, arm_tag="left") + if last_qpos is None: + now_qpos = self.left_entity.get_qpos() + else: + now_qpos = deepcopy(last_qpos) + + trans_target_pose = self._trans_from_gripper_to_endlink(target_pose, arm_tag="left") + + if self.communication_flag: + self.left_conn.send({ + "cmd": "plan_path", + "qpos": now_qpos, + "target_pose": trans_target_pose, + "constraint_pose": constraint_pose, + "arms_tag": "left", + }) + return self.left_conn.recv() + else: + return self.left_planner.plan_path( + now_qpos, + trans_target_pose, + constraint_pose=constraint_pose, + arms_tag="left", + ) + + def right_plan_path( + self, + target_pose, + constraint_pose=None, + use_point_cloud=False, + use_attach=False, + last_qpos=None, + ): + if constraint_pose is not None: + constraint_pose = self.get_constraint_pose(constraint_pose, arm_tag="right") + if last_qpos is None: + now_qpos = self.right_entity.get_qpos() + else: + now_qpos = deepcopy(last_qpos) + + trans_target_pose = self._trans_from_gripper_to_endlink(target_pose, arm_tag="right") + + if self.communication_flag: + self.right_conn.send({ + "cmd": "plan_path", + "qpos": now_qpos, + "target_pose": trans_target_pose, + "constraint_pose": constraint_pose, + "arms_tag": "right", + }) + return self.right_conn.recv() + else: + return self.right_planner.plan_path( + now_qpos, + trans_target_pose, + constraint_pose=constraint_pose, + arms_tag="right", + ) + + # The data of gripper has been normalized + def get_left_arm_jointState(self) -> list: + jointState_list = [] + for joint in self.left_arm_joints: + jointState_list.append(joint.get_drive_target()[0].astype(float)) + jointState_list.append(self.get_left_gripper_val()) + return jointState_list + + def get_right_arm_jointState(self) -> list: + jointState_list = [] + for joint in self.right_arm_joints: + jointState_list.append(joint.get_drive_target()[0].astype(float)) + jointState_list.append(self.get_right_gripper_val()) + return jointState_list + + def get_left_arm_real_jointState(self) -> list: + jointState_list = [] + left_joints_qpos = self.left_entity.get_qpos() + left_active_joints = self.left_entity.get_active_joints() + for joint in self.left_arm_joints: + jointState_list.append(left_joints_qpos[left_active_joints.index(joint)]) + jointState_list.append(self.get_left_gripper_val()) + return jointState_list + + def get_right_arm_real_jointState(self) -> list: + jointState_list = [] + right_joints_qpos = self.right_entity.get_qpos() + right_active_joints = self.right_entity.get_active_joints() + for joint in self.right_arm_joints: + jointState_list.append(right_joints_qpos[right_active_joints.index(joint)]) + jointState_list.append(self.get_right_gripper_val()) + return jointState_list + + def get_left_gripper_val(self): + if None in self.left_gripper: + print("No gripper") + return 0 + return self.left_gripper_val + + def get_right_gripper_val(self): + if None in self.right_gripper: + print("No gripper") + return 0 + return self.right_gripper_val + + def is_left_gripper_open(self): + return self.left_gripper_val > 0.8 + + def is_right_gripper_open(self): + return self.right_gripper_val > 0.8 + + def is_left_gripper_open_half(self): + return self.left_gripper_val > 0.45 + + def is_right_gripper_open_half(self): + return self.right_gripper_val > 0.45 + + def is_left_gripper_close(self): + return self.left_gripper_val < 0.2 + + def is_right_gripper_close(self): + return self.right_gripper_val < 0.2 + + # get move group joint pose + def get_left_ee_pose(self): + return self._trans_endpose(arm_tag="left", is_endpose=False) + + def get_right_ee_pose(self): + return self._trans_endpose(arm_tag="right", is_endpose=False) + + # get gripper centor pose + def get_left_tcp_pose(self): + return self._trans_endpose(arm_tag="left", is_endpose=True) + + def get_right_tcp_pose(self): + return self._trans_endpose(arm_tag="right", is_endpose=True) + + def get_left_orig_endpose(self): + pose = self.left_ee.global_pose + global_trans_matrix = self.left_global_trans_matrix + pose.p = pose.p - self.left_entity_origion_pose.p + pose.p = t3d.quaternions.quat2mat(self.left_entity_origion_pose.q).T @ pose.p + return (pose.p.tolist() + t3d.quaternions.mat2quat( + t3d.quaternions.quat2mat(self.left_entity_origion_pose.q).T @ t3d.quaternions.quat2mat(pose.q) + @ global_trans_matrix).tolist()) + + def get_right_orig_endpose(self): + pose = self.right_ee.global_pose + global_trans_matrix = self.right_global_trans_matrix + pose.p = pose.p - self.right_entity_origion_pose.p + pose.p = t3d.quaternions.quat2mat(self.right_entity_origion_pose.q).T @ pose.p + return (pose.p.tolist() + t3d.quaternions.mat2quat( + t3d.quaternions.quat2mat(self.right_entity_origion_pose.q).T @ t3d.quaternions.quat2mat(pose.q) + @ global_trans_matrix).tolist()) + + def _trans_endpose(self, arm_tag=None, is_endpose=False): + if arm_tag is None: + print("No arm tag") + return + gripper_bias = (self.left_gripper_bias if arm_tag == "left" else self.right_gripper_bias) + global_trans_matrix = (self.left_global_trans_matrix if arm_tag == "left" else self.right_global_trans_matrix) + delta_matrix = (self.left_delta_matrix if arm_tag == "left" else self.right_delta_matrix) + ee_pose = (self.left_ee.global_pose if arm_tag == "left" else self.right_ee.global_pose) + endpose_arr = np.eye(4) + endpose_arr[:3, :3] = (t3d.quaternions.quat2mat(ee_pose.q) @ global_trans_matrix @ delta_matrix) + dis = gripper_bias + if is_endpose == False: + dis -= 0.12 + endpose_arr[:3, 3] = ee_pose.p + endpose_arr[:3, :3] @ np.array([dis, 0, 0]).T + res = (endpose_arr[:3, 3].tolist() + t3d.quaternions.mat2quat(endpose_arr[:3, :3]).tolist()) + return res + + def _entity_qf(self, entity): + qf = entity.compute_passive_force(gravity=True, coriolis_and_centrifugal=True) + entity.set_qf(qf) + + def set_arm_joints(self, target_position, target_velocity, arm_tag): + self._entity_qf(self.left_entity) + self._entity_qf(self.right_entity) + + joint_lst = self.left_arm_joints if arm_tag == "left" else self.right_arm_joints + for j in range(len(joint_lst)): + joint = joint_lst[j] + joint.set_drive_target(target_position[j]) + joint.set_drive_velocity_target(target_velocity[j]) + + def get_normal_real_gripper_val(self): + normal_left_gripper_val = (self.left_gripper[0][0].get_drive_target()[0] - self.left_gripper_scale[0]) / ( + self.left_gripper_scale[1] - self.left_gripper_scale[0]) + normal_right_gripper_val = (self.right_gripper[0][0].get_drive_target()[0] - self.right_gripper_scale[0]) / ( + self.right_gripper_scale[1] - self.right_gripper_scale[0]) + normal_left_gripper_val = np.clip(normal_left_gripper_val, 0, 1) + normal_right_gripper_val = np.clip(normal_right_gripper_val, 0, 1) + return [normal_left_gripper_val, normal_right_gripper_val] + + def set_gripper(self, gripper_val, arm_tag, gripper_eps=0.1): # gripper_val in [0,1] + self._entity_qf(self.left_entity) + self._entity_qf(self.right_entity) + gripper_val = np.clip(gripper_val, 0, 1) + + if arm_tag == "left": + joints = self.left_gripper + self.left_gripper_val = gripper_val + gripper_scale = self.left_gripper_scale + real_gripper_val = self.get_normal_real_gripper_val()[0] + else: + joints = self.right_gripper + self.right_gripper_val = gripper_val + gripper_scale = self.right_gripper_scale + real_gripper_val = self.get_normal_real_gripper_val()[1] + + if not joints: + print("No gripper") + return + + if (gripper_val - real_gripper_val > gripper_eps + and gripper_eps > 0) or (gripper_val - real_gripper_val < gripper_eps and gripper_eps < 0): + gripper_val = real_gripper_val + gripper_eps # TODO + + real_gripper_val = gripper_scale[0] + gripper_val * (gripper_scale[1] - gripper_scale[0]) + + for joint in joints: + real_joint: sapien.physx.PhysxArticulationJoint = joint[0] + drive_target = real_gripper_val * joint[1] + joint[2] + drive_velocity_target = (np.clip(drive_target - real_joint.drive_target, -1.0, 1.0) * 0.05) + real_joint.set_drive_target(drive_target) + real_joint.set_drive_velocity_target(drive_velocity_target) + + +def planner_process_worker(conn, args): + import os + from .planner import CuroboPlanner # 或者绝对路径导入 + + planner = CuroboPlanner(args["origin_pose"], args["joints_name"], args["all_joints"], yml_path=args["yml_path"]) + + while True: + try: + msg = conn.recv() + if msg["cmd"] == "plan_path": + result = planner.plan_path( + msg["qpos"], + msg["target_pose"], + constraint_pose=msg.get("constraint_pose", None), + arms_tag=msg["arms_tag"], + ) + conn.send(result) + + elif msg["cmd"] == "plan_batch": + result = planner.plan_batch( + msg["qpos"], + msg["target_pose_list"], + constraint_pose=msg.get("constraint_pose", None), + arms_tag=msg["arms_tag"], + ) + conn.send(result) + + elif msg["cmd"] == "plan_grippers": + result = planner.plan_grippers( + msg["now_val"], + msg["target_val"], + ) + conn.send(result) + + elif msg["cmd"] == "update_point_cloud": + planner.update_point_cloud(msg["pcd"], resolution=msg.get("resolution", 0.02)) + conn.send("ok") + + elif msg["cmd"] == "reset": + planner.motion_gen.reset(reset_seed=True) + conn.send("ok") + + elif msg["cmd"] == "exit": + conn.close() + break + + else: + conn.send({"error": f"Unknown command {msg['cmd']}"}) + + except EOFError: + break + except Exception as e: + conn.send({"error": str(e)}) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/rotate_qrcode.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/rotate_qrcode.py new file mode 100644 index 00000000..c2abd061 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/rotate_qrcode.py @@ -0,0 +1,78 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +from copy import deepcopy + + +class rotate_qrcode(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + qrcode_pose = rand_pose( + xlim=[-0.25, 0.25], + ylim=[-0.2, 0.0], + qpos=[0, 0, 0.707, 0.707], + rotate_rand=True, + rotate_lim=[0, 0.7, 0], + ) + while abs(qrcode_pose.p[0]) < 0.05: + qrcode_pose = rand_pose( + xlim=[-0.25, 0.25], + ylim=[-0.2, 0.0], + qpos=[0, 0, 0.707, 0.707], + rotate_rand=True, + rotate_lim=[0, 0.7, 0], + ) + + self.model_id = np.random.choice([0, 1, 2, 3], 1)[0] + self.qrcode = create_actor( + self, + pose=qrcode_pose, + modelname="070_paymentsign", + convex=True, + model_id=self.model_id, + ) + + self.add_prohibit_area(self.qrcode, padding=0.12) + # Define target placement position based on arm tag (left or right side of table) + target_x = -0.2 if self.qrcode.get_pose().p[0] < 0 else 0.2 + self.target_pose = [target_x, -0.15, 0.74 + self.table_z_bias, 1, 0, 0, 0] + + def play_once(self): + # Determine which arm to use based on QR code position (left if on left side, right otherwise) + arm_tag = ArmTag("left" if self.qrcode.get_pose().p[0] < 0 else "right") + + # Grasp the QR code with specified pre-grasp distance + self.move(self.grasp_actor(self.qrcode, arm_tag=arm_tag, pre_grasp_dis=0.05)) + + # Lift the QR code vertically by 0.07 meters + self.move(self.move_by_displacement(arm_tag=arm_tag, z=0.07)) + + # Place the QR code at the target position with specified placement parameters + self.move( + self.place_actor( + self.qrcode, + arm_tag=arm_tag, + target_pose=self.target_pose, + pre_dis=0.07, + dis=0.01, + constrain="align", + )) + + self.info["info"] = { + "{A}": f"070_paymentsign/base{self.model_id}", + "{a}": str(arm_tag), + } + return self.info + + def check_success(self): + qrcode_quat = self.qrcode.get_pose().q + qrcode_pos = self.qrcode.get_pose().p + target_quat = [0.707, 0.707, 0, 0] + if qrcode_quat[0] < 0: + qrcode_quat = qrcode_quat * -1 + eps = 0.05 + return (np.all(np.abs(qrcode_quat - target_quat) < eps) and qrcode_pos[2] < 0.75 + self.table_z_bias + and self.is_left_gripper_open() and self.is_right_gripper_open()) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/scan_object.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/scan_object.py new file mode 100644 index 00000000..005d6aa5 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/scan_object.py @@ -0,0 +1,112 @@ +from ._base_task import Base_Task +from .utils import * +from ._GLOBAL_CONFIGS import * + + +class scan_object(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + tag = np.random.randint(2) + if tag == 0: + scanner_x_lim = [-0.25, -0.05] + object_x_lim = [0.05, 0.25] + else: + scanner_x_lim = [0.05, 0.25] + object_x_lim = [-0.25, -0.05] + scanner_pose = rand_pose( + xlim=scanner_x_lim, + ylim=[-0.15, -0.05], + qpos=[0, 0, 0.707, 0.707], + rotate_rand=True, + rotate_lim=[0, 1.2, 0], + ) + self.scanner_id = np.random.choice([0, 1, 2, 3, 4], 1)[0] + self.scanner = create_actor( + scene=self.scene, + pose=scanner_pose, + modelname="024_scanner", + convex=True, + model_id=self.scanner_id, + ) + + object_pose = rand_pose( + xlim=object_x_lim, + ylim=[-0.2, 0.0], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=True, + rotate_lim=[0, 1.2, 0], + ) + self.object_id = np.random.choice([0, 1, 2, 3, 4, 5], 1)[0] + self.object = create_actor( + scene=self.scene, + pose=object_pose, + modelname="112_tea-box", + convex=True, + model_id=self.object_id, + ) + self.add_prohibit_area(self.scanner, padding=0.1) + self.add_prohibit_area(self.object, padding=0.1) + target_posi = [-0.2, -0.03, 0.2, -0.01] + self.prohibited_area.append(target_posi) + self.left_object_target_pose = [-0.03, -0.02, 0.95, 0.707, 0, -0.707, 0] + self.right_object_target_pose = [0.03, -0.02, 0.95, 0.707, 0, 0.707, 0] + + def play_once(self): + scanner_arm_tag = ArmTag("left" if self.scanner.get_pose().p[0] < 0 else "right") + object_arm_tag = scanner_arm_tag.opposite + + # Move the scanner and object to the gripper + self.move( + self.grasp_actor(self.scanner, arm_tag=scanner_arm_tag, pre_grasp_dis=0.08), + self.grasp_actor(self.object, arm_tag=object_arm_tag, pre_grasp_dis=0.08), + ) + self.move( + self.move_by_displacement(arm_tag=scanner_arm_tag, x=0.05 if scanner_arm_tag == "right" else -0.05, z=0.13), + self.move_by_displacement(arm_tag=object_arm_tag, x=0.05 if object_arm_tag == "right" else -0.05, z=0.13), + ) + # Get object target pose and place the object + object_target_pose = (self.right_object_target_pose + if object_arm_tag == "right" else self.left_object_target_pose) + self.move( + self.place_actor( + self.object, + arm_tag=object_arm_tag, + target_pose=object_target_pose, + pre_dis=0.0, + dis=0.0, + is_open=False, + )) + + # Move the scanner to align with the object + self.move( + self.place_actor( + self.scanner, + arm_tag=scanner_arm_tag, + target_pose=self.object.get_functional_point(1), + functional_point_id=0, + pre_dis=0.05, + dis=0.05, + is_open=False, + )) + + self.info["info"] = { + "{A}": f"112_tea-box/base{self.object_id}", + "{B}": f"024_scanner/base{self.scanner_id}", + "{a}": str(object_arm_tag), + "{b}": str(scanner_arm_tag), + } + return self.info + + def check_success(self): + object_pose = self.object.get_pose().p + scanner_func_pose = self.scanner.get_functional_point(0) + target_vec = t3d.quaternions.quat2mat(scanner_func_pose[-4:]) @ np.array([0, 0, -1]) + obj2scanner_vec = scanner_func_pose[:3] - object_pose + dis = np.sum(target_vec * obj2scanner_vec) + object_pose1 = object_pose + dis * target_vec + eps = 0.025 + return (np.all(np.abs(object_pose1 - scanner_func_pose[:3]) < eps) and dis > 0 and dis < 0.07 + and self.is_left_gripper_close() and self.is_right_gripper_close()) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/shake_bottle.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/shake_bottle.py new file mode 100644 index 00000000..4e281004 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/shake_bottle.py @@ -0,0 +1,84 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math + + +class shake_bottle(Base_Task): + + def setup_demo(self, is_test=False, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + self.id_list = [i for i in range(20)] + rand_pos = rand_pose( + xlim=[-0.15, 0.15], + ylim=[-0.15, -0.05], + zlim=[0.785], + qpos=[0, 0, 1, 0], + rotate_rand=True, + rotate_lim=[0, 0, np.pi / 4], + ) + while abs(rand_pos.p[0]) < 0.1: + rand_pos = rand_pose( + xlim=[-0.15, 0.15], + ylim=[-0.15, -0.05], + zlim=[0.785], + qpos=[0, 0, 1, 0], + rotate_rand=True, + rotate_lim=[0, 0, np.pi / 4], + ) + self.bottle_id = np.random.choice(self.id_list) + self.bottle = create_actor( + scene=self, + pose=rand_pos, + modelname="001_bottle", + convex=True, + model_id=self.bottle_id, + ) + self.bottle.set_mass(0.01) + self.add_prohibit_area(self.bottle, padding=0.05) + + def play_once(self): + # Determine which arm to use based on bottle position + arm_tag = ArmTag("right" if self.bottle.get_pose().p[0] > 0 else "left") + + # Grasp the bottle with specified pre-grasp distance + self.move(self.grasp_actor(self.bottle, arm_tag=arm_tag, pre_grasp_dis=0.1)) + + # Lift the bottle up by 0.2m while rotating to target orientation + target_quat = [0.707, 0, 0, 0.707] + self.move(self.move_by_displacement(arm_tag=arm_tag, z=0.1, quat=target_quat)) + + # Prepare two shaking orientations by rotating around y-axis + quat1 = deepcopy(target_quat) + quat2 = deepcopy(target_quat) + # First shake rotation (7π/8 around y-axis) + y_rotation = t3d.euler.euler2quat(0, (np.pi / 8) * 7, 0) + rotated_q = t3d.quaternions.qmult(y_rotation, quat1) + quat1 = [-rotated_q[1], rotated_q[0], rotated_q[3], -rotated_q[2]] + + # Second shake rotation (-7π/8 around y-axis) + y_rotation = t3d.euler.euler2quat(0, -7 * (np.pi / 8), 0) + rotated_q = t3d.quaternions.qmult(y_rotation, quat2) + quat2 = [-rotated_q[1], rotated_q[0], rotated_q[3], -rotated_q[2]] + + # Perform shaking motion three times (alternating between two orientations) + for _ in range(3): + # Move up with first shaking orientation + self.move(self.move_by_displacement(arm_tag=arm_tag, z=0.05, quat=quat1)) + # Move down with second shaking orientation + self.move(self.move_by_displacement(arm_tag=arm_tag, z=-0.05, quat=quat2)) + + # Return to original grasp orientation + self.move(self.move_by_displacement(arm_tag=arm_tag, quat=target_quat)) + + self.info["info"] = { + "{A}": f"001_bottle/base{self.bottle_id}", + "{a}": str(arm_tag), + } + return self.info + + def check_success(self): + bottle_pose = self.bottle.get_pose().p + return bottle_pose[2] > 0.8 + self.table_z_bias diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/shake_bottle_horizontally.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/shake_bottle_horizontally.py new file mode 100644 index 00000000..00be16b5 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/shake_bottle_horizontally.py @@ -0,0 +1,94 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math + + +class shake_bottle_horizontally(Base_Task): + + def setup_demo(self, is_test=False, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + self.id_list = [i for i in range(20)] + rand_pos = rand_pose( + xlim=[-0.15, 0.15], + ylim=[-0.15, -0.05], + zlim=[0.785], + qpos=[0, 0, 1, 0], + rotate_rand=True, + rotate_lim=[0, 0, np.pi / 4], + ) + while abs(rand_pos.p[0]) < 0.1: + rand_pos = rand_pose( + xlim=[-0.15, 0.15], + ylim=[-0.15, -0.05], + zlim=[0.785], + qpos=[0, 0, 1, 0], + rotate_rand=True, + rotate_lim=[0, 0, np.pi / 4], + ) + self.bottle_id = np.random.choice(self.id_list) + self.bottle = create_actor( + scene=self, + pose=rand_pos, + modelname="001_bottle", + convex=True, + model_id=self.bottle_id, + ) + self.bottle.set_mass(0.01) + self.add_prohibit_area(self.bottle, padding=0.05) + + def play_once(self): + # Determine which arm to use based on bottle position + arm_tag = ArmTag("right" if self.bottle.get_pose().p[0] > 0 else "left") + + # Grasp the bottle with specified pre-grasp distance + self.move(self.grasp_actor(self.bottle, arm_tag=arm_tag, pre_grasp_dis=0.1)) + + # Lift the bottle up while setting initial orientation (quaternion) + target_quat = [0.707, 0, 0, 0.707] + self.move(self.move_by_displacement(arm_tag=arm_tag, z=0.1, quat=target_quat)) + + # Rotate the bottle 90 degrees around y-axis (π/2) + y_rotation = t3d.euler.euler2quat(0, (np.pi / 2), 0) + rotated_q = t3d.quaternions.qmult(y_rotation, target_quat) + target_quat = [-rotated_q[1], rotated_q[0], rotated_q[3], -rotated_q[2]] + self.move(self.move_by_displacement(arm_tag=arm_tag, quat=target_quat)) + + # Prepare two quaternions for shaking motion + quat1 = deepcopy(target_quat) + quat2 = deepcopy(target_quat) + # First shake rotation: 157.5 degrees (7π/8) clockwise around y-axis + y_rotation = t3d.euler.euler2quat(0, (np.pi / 8) * 7, 0) + rotated_q = t3d.quaternions.qmult(y_rotation, quat1) + quat1 = [-rotated_q[1], rotated_q[0], rotated_q[3], -rotated_q[2]] + + # Second shake rotation: 157.5 degrees (7π/8) counter-clockwise around y-axis + y_rotation = t3d.euler.euler2quat(0, -7 * (np.pi / 8), 0) + rotated_q = t3d.quaternions.qmult(y_rotation, quat2) + quat2 = [-rotated_q[1], rotated_q[0], rotated_q[3], -rotated_q[2]] + # Move up with first rotation + self.move(self.move_by_displacement(arm_tag=arm_tag, z=0., quat=quat1)) + # Move down with opposite rotation + self.move(self.move_by_displacement(arm_tag=arm_tag, z=-0.03, quat=quat2)) + + # Perform shaking motion (up and down while rotating) three times + for _ in range(2): + # Move up with first rotation + self.move(self.move_by_displacement(arm_tag=arm_tag, z=0.05, quat=quat1)) + # Move down with opposite rotation + self.move(self.move_by_displacement(arm_tag=arm_tag, z=-0.05, quat=quat2)) + + # Return to the orientation after initial tilt (before shaking) + self.move(self.move_by_displacement(arm_tag=arm_tag, quat=target_quat)) + + self.info["info"] = { + "{A}": f"001_bottle/base{self.bottle_id}", + "{a}": str(arm_tag), + } + return self.info + + def check_success(self): + bottle_pose = self.bottle.get_pose().p + return bottle_pose[2] > 0.8 + self.table_z_bias diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/stack_blocks_three.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/stack_blocks_three.py new file mode 100644 index 00000000..0ab0917f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/stack_blocks_three.py @@ -0,0 +1,130 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math + + +class stack_blocks_three(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + block_half_size = 0.025 + block_pose_lst = [] + for i in range(3): + block_pose = rand_pose( + xlim=[-0.28, 0.28], + ylim=[-0.08, 0.05], + zlim=[0.741 + block_half_size], + qpos=[1, 0, 0, 0], + ylim_prop=True, + rotate_rand=True, + rotate_lim=[0, 0, 0.75], + ) + + def check_block_pose(block_pose): + for j in range(len(block_pose_lst)): + if (np.sum(pow(block_pose.p[:2] - block_pose_lst[j].p[:2], 2)) < 0.01): + return False + return True + + while (abs(block_pose.p[0]) < 0.05 or np.sum(pow(block_pose.p[:2] - np.array([0, -0.1]), 2)) < 0.0225 + or not check_block_pose(block_pose)): + block_pose = rand_pose( + xlim=[-0.28, 0.28], + ylim=[-0.08, 0.05], + zlim=[0.741 + block_half_size], + qpos=[1, 0, 0, 0], + ylim_prop=True, + rotate_rand=True, + rotate_lim=[0, 0, 0.75], + ) + block_pose_lst.append(deepcopy(block_pose)) + + def create_block(block_pose, color): + return create_box( + scene=self, + pose=block_pose, + half_size=(block_half_size, block_half_size, block_half_size), + color=color, + name="box", + ) + + self.block1 = create_block(block_pose_lst[0], (1, 0, 0)) + self.block2 = create_block(block_pose_lst[1], (0, 1, 0)) + self.block3 = create_block(block_pose_lst[2], (0, 0, 1)) + self.add_prohibit_area(self.block1, padding=0.05) + self.add_prohibit_area(self.block2, padding=0.05) + self.add_prohibit_area(self.block3, padding=0.05) + target_pose = [-0.04, -0.13, 0.04, -0.05] + self.prohibited_area.append(target_pose) + self.block1_target_pose = [0, -0.13, 0.75 + self.table_z_bias, 0, 1, 0, 0] + + def play_once(self): + # Initialize tracking variables for last used gripper and actor + self.last_gripper = None + self.last_actor = None + + # Pick and place the first block (red) and get which arm was used + arm_tag1 = self.pick_and_place_block(self.block1) + # Pick and place the second block (green) and get which arm was used + arm_tag2 = self.pick_and_place_block(self.block2) + # Pick and place the third block (blue) and get which arm was used + arm_tag3 = self.pick_and_place_block(self.block3) + + # Store information about the blocks and which arms were used + self.info["info"] = { + "{A}": "red block", + "{B}": "green block", + "{C}": "blue block", + "{a}": str(arm_tag1), + "{b}": str(arm_tag2), + "{c}": str(arm_tag3), + } + return self.info + + def pick_and_place_block(self, block: Actor): + block_pose = block.get_pose().p + arm_tag = ArmTag("left" if block_pose[0] < 0 else "right") + + if self.last_gripper is not None and (self.last_gripper != arm_tag): + self.move( + self.grasp_actor(block, arm_tag=arm_tag, pre_grasp_dis=0.09), # arm_tag + self.back_to_origin(arm_tag=arm_tag.opposite), # arm_tag.opposite + ) + else: + self.move(self.grasp_actor(block, arm_tag=arm_tag, pre_grasp_dis=0.09)) # arm_tag + + self.move(self.move_by_displacement(arm_tag=arm_tag, z=0.07)) # arm_tag + + if self.last_actor is None: + target_pose = [0, -0.13, 0.75 + self.table_z_bias, 0, 1, 0, 0] + else: + target_pose = self.last_actor.get_functional_point(1) + + self.move( + self.place_actor( + block, + target_pose=target_pose, + arm_tag=arm_tag, + functional_point_id=0, + pre_dis=0.05, + dis=0., + pre_dis_axis="fp", + )) + self.move(self.move_by_displacement(arm_tag=arm_tag, z=0.07)) # arm_tag + + self.last_gripper = arm_tag + self.last_actor = block + return str(arm_tag) + + def check_success(self): + block1_pose = self.block1.get_pose().p + block2_pose = self.block2.get_pose().p + block3_pose = self.block3.get_pose().p + eps = [0.025, 0.025, 0.012] + + return (np.all(abs(block2_pose - np.array(block1_pose[:2].tolist() + [block1_pose[2] + 0.05])) < eps) + and np.all(abs(block3_pose - np.array(block2_pose[:2].tolist() + [block2_pose[2] + 0.05])) < eps) + and self.is_left_gripper_open() and self.is_right_gripper_open()) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/stack_blocks_two.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/stack_blocks_two.py new file mode 100644 index 00000000..bb7e0c82 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/stack_blocks_two.py @@ -0,0 +1,122 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math + + +class stack_blocks_two(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + block_half_size = 0.025 + block_pose_lst = [] + for i in range(2): + block_pose = rand_pose( + xlim=[-0.28, 0.28], + ylim=[-0.08, 0.05], + zlim=[0.741 + block_half_size], + qpos=[1, 0, 0, 0], + ylim_prop=True, + rotate_rand=True, + rotate_lim=[0, 0, 0.75], + ) + + def check_block_pose(block_pose): + for j in range(len(block_pose_lst)): + if (np.sum(pow(block_pose.p[:2] - block_pose_lst[j].p[:2], 2)) < 0.01): + return False + return True + + while (abs(block_pose.p[0]) < 0.05 or np.sum(pow(block_pose.p[:2] - np.array([0, -0.1]), 2)) < 0.0225 + or not check_block_pose(block_pose)): + block_pose = rand_pose( + xlim=[-0.28, 0.28], + ylim=[-0.08, 0.05], + zlim=[0.741 + block_half_size], + qpos=[1, 0, 0, 0], + ylim_prop=True, + rotate_rand=True, + rotate_lim=[0, 0, 0.75], + ) + block_pose_lst.append(deepcopy(block_pose)) + + def create_block(block_pose, color): + return create_box( + scene=self, + pose=block_pose, + half_size=(block_half_size, block_half_size, block_half_size), + color=color, + name="box", + ) + + self.block1 = create_block(block_pose_lst[0], (1, 0, 0)) + self.block2 = create_block(block_pose_lst[1], (0, 1, 0)) + self.add_prohibit_area(self.block1, padding=0.07) + self.add_prohibit_area(self.block2, padding=0.07) + target_pose = [-0.04, -0.13, 0.04, -0.05] + self.prohibited_area.append(target_pose) + self.block1_target_pose = [0, -0.13, 0.75 + self.table_z_bias, 0, 1, 0, 0] + + def play_once(self): + # Initialize tracking variables for gripper and actor + self.last_gripper = None + self.last_actor = None + + # Pick and place the first block (block1) and get its arm tag + arm_tag1 = self.pick_and_place_block(self.block1) + # Pick and place the second block (block2) and get its arm tag + arm_tag2 = self.pick_and_place_block(self.block2) + + # Store information about the blocks and their associated arms + self.info["info"] = { + "{A}": "red block", + "{B}": "green block", + "{a}": arm_tag1, + "{b}": arm_tag2, + } + return self.info + + def pick_and_place_block(self, block: Actor): + block_pose = block.get_pose().p + arm_tag = ArmTag("left" if block_pose[0] < 0 else "right") + + if self.last_gripper is not None and (self.last_gripper != arm_tag): + self.move( + self.grasp_actor(block, arm_tag=arm_tag, pre_grasp_dis=0.09), # arm_tag + self.back_to_origin(arm_tag=arm_tag.opposite), # arm_tag.opposite + ) + else: + self.move(self.grasp_actor(block, arm_tag=arm_tag, pre_grasp_dis=0.09)) # arm_tag + + self.move(self.move_by_displacement(arm_tag=arm_tag, z=0.07)) # arm_tag + + if self.last_actor is None: + target_pose = [0, -0.13, 0.75 + self.table_z_bias, 0, 1, 0, 0] + else: + target_pose = self.last_actor.get_functional_point(1) + + self.move( + self.place_actor( + block, + target_pose=target_pose, + arm_tag=arm_tag, + functional_point_id=0, + pre_dis=0.05, + dis=0., + pre_dis_axis="fp", + )) + self.move(self.move_by_displacement(arm_tag=arm_tag, z=0.07)) # arm_tag + + self.last_gripper = arm_tag + self.last_actor = block + return str(arm_tag) + + def check_success(self): + block1_pose = self.block1.get_pose().p + block2_pose = self.block2.get_pose().p + eps = [0.025, 0.025, 0.012] + + return (np.all(abs(block2_pose - np.array(block1_pose[:2].tolist() + [block1_pose[2] + 0.05])) < eps) + and self.is_left_gripper_open() and self.is_right_gripper_open()) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/stack_bowls_three.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/stack_bowls_three.py new file mode 100644 index 00000000..ae467d35 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/stack_bowls_three.py @@ -0,0 +1,123 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math + + +class stack_bowls_three(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + bowl_pose_lst = [] + for i in range(3): + bowl_pose = rand_pose( + xlim=[-0.3, 0.3], + ylim=[-0.15, 0.15], + qpos=[0.5, 0.5, 0.5, 0.5], + ylim_prop=True, + rotate_rand=False, + ) + + def check_bowl_pose(bowl_pose): + for j in range(len(bowl_pose_lst)): + if (np.sum(pow(bowl_pose.p[:2] - bowl_pose_lst[j].p[:2], 2)) < 0.0169): + return False + return True + + while (abs(bowl_pose.p[0]) < 0.09 or np.sum(pow(bowl_pose.p[:2] - np.array([0, -0.1]), 2)) < 0.0169 + or not check_bowl_pose(bowl_pose)): + bowl_pose = rand_pose( + xlim=[-0.3, 0.3], + ylim=[-0.15, 0.15], + qpos=[0.5, 0.5, 0.5, 0.5], + ylim_prop=True, + rotate_rand=False, + ) + bowl_pose_lst.append(deepcopy(bowl_pose)) + + bowl_pose_lst = sorted(bowl_pose_lst, key=lambda x: x.p[1]) + + def create_bowl(bowl_pose): + return create_actor(self, pose=bowl_pose, modelname="002_bowl", model_id=3, convex=True) + + self.bowl1 = create_bowl(bowl_pose_lst[0]) + self.bowl2 = create_bowl(bowl_pose_lst[1]) + self.bowl3 = create_bowl(bowl_pose_lst[2]) + + self.add_prohibit_area(self.bowl1, padding=0.07) + self.add_prohibit_area(self.bowl2, padding=0.07) + self.add_prohibit_area(self.bowl3, padding=0.07) + target_pose = [-0.1, -0.15, 0.1, -0.05] + self.prohibited_area.append(target_pose) + self.bowl1_target_pose = np.array([0, -0.1, 0.76]) + self.quat_of_target_pose = [0, 0.707, 0.707, 0] + + def move_bowl(self, actor, target_pose): + actor_pose = actor.get_pose().p + arm_tag = ArmTag("left" if actor_pose[0] < 0 else "right") + + if self.las_arm is None or arm_tag == self.las_arm: + self.move( + self.grasp_actor( + actor, + arm_tag=arm_tag, + contact_point_id=[0, 2][int(arm_tag == "left")], + pre_grasp_dis=0.1, + )) + else: + self.move( + self.grasp_actor( + actor, + arm_tag=arm_tag, + contact_point_id=[0, 2][int(arm_tag == "left")], + pre_grasp_dis=0.1, + ), # arm_tag + self.back_to_origin(arm_tag=arm_tag.opposite), # arm_tag.opposite + ) + self.move(self.move_by_displacement(arm_tag, z=0.1)) + self.move( + self.place_actor( + actor, + target_pose=target_pose.tolist() + self.quat_of_target_pose, + arm_tag=arm_tag, + functional_point_id=0, + pre_dis=0.09, + dis=0, + constrain="align", + )) + self.move(self.move_by_displacement(arm_tag, z=0.09)) + self.las_arm = arm_tag + return arm_tag + + def play_once(self): + # Initialize last arm used to None + self.las_arm = None + + # Move bowl1 to position [0, -0.1, 0.76] + self.move_bowl(self.bowl1, self.bowl1_target_pose) + # Move bowl2 to be 0.05m above bowl1's position + self.move_bowl(self.bowl2, self.bowl1.get_pose().p + [0, 0, 0.05]) + # Move bowl3 to be 0.05m above bowl2's position + self.move_bowl(self.bowl3, self.bowl2.get_pose().p + [0, 0, 0.05]) + + self.info["info"] = {"{A}": f"002_bowl/base3"} + return self.info + + def check_success(self): + bowl1_pose = self.bowl1.get_pose().p + bowl2_pose = self.bowl2.get_pose().p + bowl3_pose = self.bowl3.get_pose().p + bowl1_pose, bowl2_pose, bowl3_pose = sorted([bowl1_pose, bowl2_pose, bowl3_pose], key=lambda x: x[2]) + target_height = [ + 0.74 + self.table_z_bias, + 0.77 + self.table_z_bias, + 0.81 + self.table_z_bias, + ] + eps = 0.02 + eps2 = 0.04 + return (np.all(abs(bowl1_pose[:2] - bowl2_pose[:2]) < eps2) + and np.all(abs(bowl2_pose[:2] - bowl3_pose[:2]) < eps2) + and np.all(np.array([bowl1_pose[2], bowl2_pose[2], bowl3_pose[2]]) - target_height < eps) + and self.is_left_gripper_open() and self.is_right_gripper_open()) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/stack_bowls_two.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/stack_bowls_two.py new file mode 100644 index 00000000..52942ded --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/stack_bowls_two.py @@ -0,0 +1,122 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math + + +class stack_bowls_two(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + bowl_pose_lst = [] + for i in range(2): + bowl_pose = rand_pose( + xlim=[-0.3, 0.3], + ylim=[-0.15, 0.15], + qpos=[0.5, 0.5, 0.5, 0.5], + ylim_prop=True, + rotate_rand=False, + ) + + def check_bowl_pose(bowl_pose): + for j in range(len(bowl_pose_lst)): + if (np.sum(pow(bowl_pose.p[:2] - bowl_pose_lst[j].p[:2], 2)) < 0.0169): + return False + return True + + while (abs(bowl_pose.p[0]) < 0.09 or np.sum(pow(bowl_pose.p[:2] - np.array([0, -0.1]), 2)) < 0.0169 + or not check_bowl_pose(bowl_pose)): + bowl_pose = rand_pose( + xlim=[-0.3, 0.3], + ylim=[-0.15, 0.15], + qpos=[0.5, 0.5, 0.5, 0.5], + ylim_prop=True, + rotate_rand=False, + ) + bowl_pose_lst.append(deepcopy(bowl_pose)) + + bowl_pose_lst = sorted(bowl_pose_lst, key=lambda x: x.p[1]) + + def create_bowl(bowl_pose): + return create_actor(self, pose=bowl_pose, modelname="002_bowl", model_id=3, convex=True) + + self.bowl1 = create_bowl(bowl_pose_lst[0]) + self.bowl2 = create_bowl(bowl_pose_lst[1]) + + self.add_prohibit_area(self.bowl1, padding=0.07) + self.add_prohibit_area(self.bowl2, padding=0.07) + target_pose = [-0.1, -0.15, 0.1, -0.05] + self.prohibited_area.append(target_pose) + self.bowl1_target_pose = np.array([0, -0.1, 0.76]) + self.quat_of_target_pose = [0, 0.707, 0.707, 0] + + def move_bowl(self, actor, target_pose): + actor_pose = actor.get_pose().p + arm_tag = ArmTag("left" if actor_pose[0] < 0 else "right") + + if self.las_arm is None or arm_tag == self.las_arm: + self.move( + self.grasp_actor( + actor, + arm_tag=arm_tag, + contact_point_id=[0, 2][int(arm_tag == "left")], + pre_grasp_dis=0.1, + )) + else: + self.move( + self.grasp_actor( + actor, + arm_tag=arm_tag, + contact_point_id=[0, 2][int(arm_tag == "left")], + pre_grasp_dis=0.1, + ), # arm_tag + self.back_to_origin(arm_tag=arm_tag.opposite), # arm_tag.opposite + ) + self.move(self.move_by_displacement(arm_tag, z=0.1)) + self.move( + self.place_actor( + actor, + target_pose=target_pose.tolist() + self.quat_of_target_pose, + arm_tag=arm_tag, + functional_point_id=0, + pre_dis=0.09, + dis=0, + constrain="align", + )) + self.move(self.move_by_displacement(arm_tag, z=0.09)) + self.las_arm = arm_tag + return arm_tag + + def play_once(self): + # Initialize last arm used to None + self.las_arm = None + + # Move bowl1 to position [0, -0.1, 0.76] + arm_tag1 = self.move_bowl(self.bowl1, self.bowl1_target_pose) + # Move bowl2 to be 0.05m above bowl1's position + arm_tag2 = self.move_bowl(self.bowl2, self.bowl1.get_pose().p + [0, 0, 0.05]) + + # Store information about the bowls and arms used in the info dictionary + self.info["info"] = { + "{A}": f"002_bowl/base3", + "{B}": f"002_bowl/base3", + "{a}": str(arm_tag1), + "{b}": str(arm_tag2), + } + return self.info + + def check_success(self): + bowl1_pose = self.bowl1.get_pose().p + bowl2_pose = self.bowl2.get_pose().p + bowl1_pose, bowl2_pose = sorted([bowl1_pose, bowl2_pose], key=lambda x: x[2]) + target_height = [ + 0.74 + self.table_z_bias, + 0.77 + self.table_z_bias, + ] + eps = 0.02 + eps2 = 0.04 + return (np.all(abs(bowl1_pose[:2] - bowl2_pose[:2]) < eps2) + and np.all(np.array([bowl1_pose[2], bowl2_pose[2]]) - target_height < eps) + and self.is_left_gripper_open() and self.is_right_gripper_open()) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/stamp_seal.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/stamp_seal.py new file mode 100644 index 00000000..4dc7ae6a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/stamp_seal.py @@ -0,0 +1,137 @@ +from ._base_task import Base_Task +from .utils import * +import sapien +import math +from ._GLOBAL_CONFIGS import * +from copy import deepcopy +import time +import numpy as np + + +class stamp_seal(Base_Task): + + def setup_demo(self, **kwags): + super()._init_task_env_(**kwags) + + def load_actors(self): + rand_pos = rand_pose( + xlim=[-0.25, 0.25], + ylim=[-0.05, 0.05], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=False, + ) + while abs(rand_pos.p[0]) < 0.05: + rand_pos = rand_pose( + xlim=[-0.25, 0.25], + ylim=[-0.05, 0.05], + qpos=[0.5, 0.5, 0.5, 0.5], + rotate_rand=False, + ) + + self.seal_id = np.random.choice([0, 2, 3, 4, 6], 1)[0] + + self.seal = create_actor( + scene=self, + pose=rand_pos, + modelname="100_seal", + convex=True, + model_id=self.seal_id, + ) + self.seal.set_mass(0.05) + + if rand_pos.p[0] > 0: + xlim = [0.05, 0.25] + else: + xlim = [-0.25, -0.05] + + target_rand_pose = rand_pose( + xlim=xlim, + ylim=[-0.05, 0.05], + qpos=[1, 0, 0, 0], + rotate_rand=False, + ) + while (np.sqrt((target_rand_pose.p[0] - rand_pos.p[0])**2 + (target_rand_pose.p[1] - rand_pos.p[1])**2) < 0.1): + target_rand_pose = rand_pose( + xlim=xlim, + ylim=[-0.05, 0.1], + qpos=[1, 0, 0, 0], + rotate_rand=False, + ) + + colors = { + "Red": (1, 0, 0), + "Green": (0, 1, 0), + "Blue": (0, 0, 1), + "Yellow": (1, 1, 0), + "Cyan": (0, 1, 1), + "Magenta": (1, 0, 1), + "Black": (0, 0, 0), + "Gray": (0.5, 0.5, 0.5), + "Orange": (1, 0.5, 0), + "Purple": (0.5, 0, 0.5), + "Brown": (0.65, 0.4, 0.16), + "Pink": (1, 0.75, 0.8), + "Lime": (0.5, 1, 0), + "Olive": (0.5, 0.5, 0), + "Teal": (0, 0.5, 0.5), + "Maroon": (0.5, 0, 0), + "Navy": (0, 0, 0.5), + "Coral": (1, 0.5, 0.31), + "Turquoise": (0.25, 0.88, 0.82), + "Indigo": (0.29, 0, 0.51), + "Beige": (0.96, 0.91, 0.81), + "Tan": (0.82, 0.71, 0.55), + "Silver": (0.75, 0.75, 0.75), + } + + color_items = list(colors.items()) + idx = np.random.choice(len(color_items)) + self.color_name, self.color_value = color_items[idx] + + half_size = [0.035, 0.035, 0.0005] + self.target = create_visual_box( + scene=self, + pose=target_rand_pose, + half_size=half_size, + color=self.color_value, + name="box", + ) + self.add_prohibit_area(self.seal, padding=0.1) + self.add_prohibit_area(self.target, padding=0.1) + self.target_pose = self.target.get_pose() + + def play_once(self): + # Determine which arm to use based on seal's position (right if on positive x-axis, else left) + arm_tag = ArmTag("right" if self.seal.get_pose().p[0] > 0 else "left") + + # Grasp the seal with specified arm, with pre-grasp distance of 0.1 + self.move(self.grasp_actor(self.seal, arm_tag=arm_tag, pre_grasp_dis=0.1, contact_point_id=[4, 5, 6, 7])) + + # Lift the seal up by 0.05 units in z-direction + self.move(self.move_by_displacement(arm_tag=arm_tag, z=0.05)) + + # Place the seal on the target pose with auto constraint and pre-placement distance of 0.1 + self.move( + self.place_actor( + self.seal, + arm_tag=arm_tag, + target_pose=self.target.get_pose(), + pre_dis=0.1, + constrain="auto", + )) + + # Update info dictionary with seal ID, color name and used arm tag + self.info["info"] = { + "{A}": f"100_seal/base{self.seal_id}", + "{B}": f"{self.color_name}", + "{a}": str(arm_tag), + } + return self.info + + def check_success(self): + seal_pose = self.seal.get_pose().p + target_pos = self.target.get_pose().p + eps1 = 0.01 + + return (np.all(abs(seal_pose[:2] - target_pos[:2]) < np.array([eps1, eps1])) + and self.robot.is_left_gripper_open() and self.robot.is_right_gripper_open()) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/turn_switch.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/turn_switch.py new file mode 100644 index 00000000..2f437afb --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/turn_switch.py @@ -0,0 +1,42 @@ +from ._base_task import Base_Task +from .utils import * + + +class turn_switch(Base_Task): + + def setup_demo(self, is_test=False, **kwargs): + super()._init_task_env_(**kwargs) + + def load_actors(self): + self.model_name = "056_switch" + self.model_id = np.random.randint(0, 8) + self.switch = rand_create_sapien_urdf_obj( + scene=self, + modelname=self.model_name, + modelid=self.model_id, + xlim=[-0.25, 0.25], + ylim=[0.0, 0.1], + zlim=[0.81, 0.84], + rotate_rand=True, + rotate_lim=[0, 0, np.pi / 4], + qpos=[0.704141, 0, 0, 0.71006], + fix_root_link=True, + ) + self.prohibited_area.append([-0.4, -0.2, 0.4, 0.2]) + + def play_once(self): + switch_pose = self.switch.get_pose() + face_dir = -switch_pose.to_transformation_matrix()[:3, 0] + arm_tag = ArmTag("right" if face_dir[0] > 0 else "left") + + # close gripper + self.move(self.close_gripper(arm_tag=arm_tag, pos=0)) + # move the gripper to turn off the switch + self.move(self.grasp_actor(self.switch, arm_tag=arm_tag, pre_grasp_dis=0.04)) + + self.info["info"] = {"{A}": f"056_switch/base{self.model_id}", "{a}": str(arm_tag)} + return self.info + + def check_success(self): + limit = self.switch.get_qlimits()[0] + return self.switch.get_qpos()[0] >= limit[1] - 0.05 diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/__init__.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/__init__.py new file mode 100644 index 00000000..52139d82 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/__init__.py @@ -0,0 +1,10 @@ +from .action import * +from .create_actor import * +from .rand_create_actor import * +from .save_file import * +from .rand_create_cluttered_actor import * +from .get_camera_config import * +from .actor_utils import * +from .transforms import * +from .pkl2hdf5 import * +from .images_to_video import * diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/action.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/action.py new file mode 100644 index 00000000..4791c1eb --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/action.py @@ -0,0 +1,88 @@ +from typing import Literal +from .transforms import _tolist +import numpy as np +import sapien + + +class ArmTag: + _instances = {} + + def __new__(cls, value): + if isinstance(value, ArmTag): + return value + if isinstance(value, str) and value in ["left", "right"]: + value = value.lower() + if value in cls._instances: + return cls._instances[value] + instance = super().__new__(cls) + cls._instances[value] = instance + return instance + raise ValueError(f"Invalid arm tag: {value}. Must be 'left' or 'right'.") + + def __init__(self, value): + if isinstance(value, str): + self.arm = value.lower() + + @property + def opposite(self): + return ArmTag("right") if self.arm == "left" else ArmTag("left") + + def __eq__(self, other): + if isinstance(other, ArmTag): + return self.arm == other.arm + if isinstance(other, str): + return self.arm == other.lower() + return False + + def __hash__(self): + return hash(self.arm) + + def __repr__(self): + return f"ArmTag('{self.arm}')" + + def __str__(self): + return self.arm + + +class Action: + arm_tag: ArmTag + action: Literal["move", "gripper"] + target_pose: list = None + target_gripper_pos: float = None + + def __init__( + self, + arm_tag: ArmTag | Literal["left", "right"], + action: Literal["move", "open", "close", "gripper"], + target_pose: sapien.Pose | list | np.ndarray = None, + target_gripper_pos: float = None, + **args, + ): + self.arm_tag = ArmTag(arm_tag) + if action != "move": + if action == "open": + self.action = "gripper" + self.target_gripper_pos = (target_gripper_pos if target_gripper_pos is not None else 1.0) + elif action == "close": + self.action = "gripper" + self.target_gripper_pos = (target_gripper_pos if target_gripper_pos is not None else 0.0) + elif action == "gripper": + self.action = "gripper" + else: + raise ValueError(f"Invalid action: {action}. Must be 'open' or 'close'.") + assert (self.target_gripper_pos is not None), "target_gripper_pos cannot be None for gripper action." + else: + self.action = "move" + assert (target_pose is not None), "target_pose cannot be None for move action." + self.target_pose = _tolist(target_pose) + self.args = args + + def __str__(self): + result = f"{self.arm_tag}: {self.action}" + if self.action == "move": + result += f"({self.target_pose})" + else: + result += f"({self.target_gripper_pos})" + if self.args: + result += f" {self.args}" + return result diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/actor_utils.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/actor_utils.py new file mode 100644 index 00000000..e0e6612d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/actor_utils.py @@ -0,0 +1,176 @@ +import sapien +import numpy as np +from copy import deepcopy +import transforms3d as t3d +from pathlib import Path + +from . import transforms +from .transforms import * + +from sapien import Entity +from sapien.physx import PhysxArticulation, PhysxArticulationLinkComponent + +from typing import Literal, Generator + + +class Actor: + POINTS = { + "contact": "contact_points_pose", + "target": "target_pose", + "functional": "functional_matrix", + "orientation": "orientation_point", + } + + def __init__(self, actor: Entity, actor_data: dict, mass=0.01): + self.actor = actor + self.config = actor_data + self.set_mass(mass) + + def get_point( + self, + type: Literal["contact", "target", "functional", "orientation"], + idx: int, + ret: Literal["matrix", "list", "pose"], + ) -> np.ndarray | list | sapien.Pose: + """Get the point of the entity actor.""" + type = self.POINTS[type] + + actor_matrix = self.actor.get_pose().to_transformation_matrix() + try: + local_matrix = np.array(self.config[type][idx]) + except: + return None + local_matrix[:3, 3] *= np.array(self.config["scale"]) + + world_matrix = actor_matrix @ local_matrix + + if ret == "matrix": + return world_matrix + elif ret == "list": + return (world_matrix[:3, 3].tolist() + t3d.quaternions.mat2quat(world_matrix[:3, :3]).tolist()) + else: + return sapien.Pose(world_matrix[:3, 3], t3d.quaternions.mat2quat(world_matrix[:3, :3])) + + def get_pose(self) -> sapien.Pose: + """Get the sapien.Pose of the actor.""" + return self.actor.get_pose() + + def get_contact_point(self, + idx: int, + ret: Literal["matrix", "list", "pose"] = "list") -> np.ndarray | list | sapien.Pose: + """Get the transformation matrix of given contact point of the actor.""" + return self.get_point("contact", idx, ret) + + def iter_contact_points( + self, + ret: Literal["matrix", "list", "pose"] = "list" + ) -> Generator[tuple[int, np.ndarray | list | sapien.Pose], None, None]: + """Iterate over all contact points of the actor.""" + for i in range(len(self.config[self.POINTS["contact"]])): + yield i, self.get_point("contact", i, ret) + + def get_functional_point(self, + idx: int, + ret: Literal["matrix", "list", "pose"] = "list") -> np.ndarray | list | sapien.Pose: + """Get the transformation matrix of given functional point of the actor.""" + return self.get_point("functional", idx, ret) + + def get_target_point(self, + idx: int, + ret: Literal["matrix", "list", "pose"] = "list") -> np.ndarray | list | sapien.Pose: + """Get the transformation matrix of given target point of the actor.""" + return self.get_point("target", idx, ret) + + def get_orientation_point(self, ret: Literal["matrix", "list", "pose"] = "list") -> np.ndarray | list | sapien.Pose: + """Get the transformation matrix of given orientation point of the actor.""" + return self.get_point("orientation", 0, ret) + + def get_name(self): + return self.actor.get_name() + + def set_name(self, name): + self.actor.set_name(name) + + def set_mass(self, mass): + for component in self.actor.get_components(): + if isinstance(component, sapien.physx.PhysxRigidDynamicComponent): + component.mass = mass + + +class ArticulationActor(Actor): + POINTS = { + "contact": "contact_points", + "target": "target_points", + "functional": "functional_points", + "orientation": "orientation_point", + } + + def __init__(self, actor: PhysxArticulation, actor_data: dict, mass=0.01): + assert isinstance(actor, PhysxArticulation), "ArticulationActor must be a Articulation" + + self.actor = actor + self.config = actor_data + + self.link_dict = self.get_link_dict() + self.set_mass(mass) + + def get_link_dict(self) -> dict[str, PhysxArticulationLinkComponent]: + link_dict = {} + for link in self.actor.get_links(): + link_dict[link.get_name()] = link + return link_dict + + def get_point( + self, + type: Literal["contact", "target", "functional", "orientation"], + idx: int, + ret: Literal["matrix", "list", "pose"], + ) -> np.ndarray | list | sapien.Pose: + """Get the point of the articulation actor.""" + type = self.POINTS[type] + local_matrix = np.array(self.config[type][idx]["matrix"]) + local_matrix[:3, 3] *= self.config["scale"] + + link = self.link_dict[self.config[type][idx]["base"]] + link_matrix = link.get_pose().to_transformation_matrix() + + world_matrix = link_matrix @ local_matrix + + if ret == "matrix": + return world_matrix + elif ret == "list": + return (world_matrix[:3, 3].tolist() + t3d.quaternions.mat2quat(world_matrix[:3, :3]).tolist()) + else: + return sapien.Pose(world_matrix[:3, 3], t3d.quaternions.mat2quat(world_matrix[:3, :3])) + + def set_mass(self, mass, links_name: list[str] = None): + for link in self.actor.get_links(): + if links_name is None or link.get_name() in links_name: + link.set_mass(mass) + + def set_properties(self, damping, stiffness, friction=None, force_limit=None): + for joint in self.actor.get_joints(): + if force_limit is not None: + joint.set_drive_properties(damping=damping, stiffness=stiffness, force_limit=force_limit) + else: + joint.set_drive_properties( + damping=damping, + stiffness=stiffness, + ) + if friction is not None: + joint.set_friction(friction) + + def set_qpos(self, qpos): + self.actor.set_qpos(qpos) + + def set_qvel(self, qvel): + self.actor.set_qvel(qvel) + + def get_qlimits(self): + return self.actor.get_qlimits() + + def get_qpos(self): + return self.actor.get_qpos() + + def get_qvel(self): + return self.actor.get_qvel() diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/create_actor.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/create_actor.py new file mode 100644 index 00000000..67c8c662 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/create_actor.py @@ -0,0 +1,654 @@ +import sapien.core as sapien +import numpy as np +from pathlib import Path +import transforms3d as t3d +import sapien.physx as sapienp +import json +import os, re + +from .actor_utils import Actor, ArticulationActor + + +class UnStableError(Exception): + + def __init__(self, msg): + super().__init__(msg) + + +def preprocess(scene, pose: sapien.Pose) -> tuple[sapien.Scene, sapien.Pose]: + """Add entity to scene. Add bias to z axis if scene is not sapien.Scene.""" + if isinstance(scene, sapien.Scene): + return scene, pose + else: + return scene.scene, sapien.Pose([pose.p[0], pose.p[1], pose.p[2] + scene.table_z_bias], pose.q) + + +# create box +def create_entity_box( + scene, + pose: sapien.Pose, + half_size, + color=None, + is_static=False, + name="", + texture_id=None, +) -> sapien.Entity: + scene, pose = preprocess(scene, pose) + + entity = sapien.Entity() + entity.set_name(name) + entity.set_pose(pose) + + # create PhysX dynamic rigid body + rigid_component = (sapien.physx.PhysxRigidDynamicComponent() + if not is_static else sapien.physx.PhysxRigidStaticComponent()) + rigid_component.attach( + sapien.physx.PhysxCollisionShapeBox(half_size=half_size, material=scene.default_physical_material)) + + # Add texture + if texture_id is not None: + + # test for both .png and .jpg + texturepath = f"./assets/background_texture/{texture_id}.png" + # create texture from file + texture2d = sapien.render.RenderTexture2D(texturepath) + material = sapien.render.RenderMaterial() + material.set_base_color_texture(texture2d) + # renderer.create_texture_from_file(texturepath) + # material.set_diffuse_texture(texturepath) + material.base_color = [1, 1, 1, 1] + material.metallic = 0.1 + material.roughness = 0.3 + else: + material = sapien.render.RenderMaterial(base_color=[*color[:3], 1]) + + # create render body for visualization + render_component = sapien.render.RenderBodyComponent() + render_component.attach( + # add a box visual shape with given size and rendering material + sapien.render.RenderShapeBox(half_size, material)) + + entity.add_component(rigid_component) + entity.add_component(render_component) + entity.set_pose(pose) + + # in general, entity should only be added to scene after it is fully built + scene.add_entity(entity) + return entity + + +def create_box( + scene, + pose: sapien.Pose, + half_size, + color=None, + is_static=False, + name="", + texture_id=None, + boxtype="default", +) -> Actor: + entity = create_entity_box( + scene=scene, + pose=pose, + half_size=half_size, + color=color, + is_static=is_static, + name=name, + texture_id=texture_id, + ) + if boxtype == "default": + data = { + "center": [0, 0, 0], + "extents": + half_size, + "scale": + half_size, + "target_pose": [[[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 1], [0, 0, 0, 1]]], + "contact_points_pose": [ + [ + [0, 0, 1, 0], + [1, 0, 0, 0], + [0, 1, 0, 0.0], + [0, 0, 0, 1], + ], # top_down(front) + [ + [1, 0, 0, 0], + [0, 0, -1, 0], + [0, 1, 0, 0.0], + [0, 0, 0, 1], + ], # top_down(right) + [ + [-1, 0, 0, 0], + [0, 0, 1, 0], + [0, 1, 0, 0.0], + [0, 0, 0, 1], + ], # top_down(left) + [ + [0, 0, -1, 0], + [-1, 0, 0, 0], + [0, 1, 0, 0.0], + [0, 0, 0, 1], + ], # top_down(back) + # [[0, 0, 1, 0], [0, -1, 0, 0], [1, 0, 0, 0.0], [0, 0, 0, 1]], # front + # [[0, -1, 0, 0], [0, 0, -1, 0], [1, 0, 0, 0.0], [0, 0, 0, 1]], # right + # [[0, 1, 0, 0], [0, 0, 1, 0], [1, 0, 0, 0.0], [0, 0, 0, 1]], # left + # [[0, 0, -1, 0], [0, 1, 0, 0], [1, 0, 0, 0.0], [0, 0, 0, 1]], # back + ], + "transform_matrix": + np.eye(4).tolist(), + "functional_matrix": [ + [ + [1.0, 0.0, 0.0, 0.0], + [0.0, -1.0, 0, 0.0], + [0.0, 0, -1.0, -1], + [0.0, 0.0, 0.0, 1.0], + ], + [ + [1.0, 0.0, 0.0, 0.0], + [0.0, -1.0, 0, 0.0], + [0.0, 0, -1.0, 1], + [0.0, 0.0, 0.0, 1.0], + ], + ], # functional points matrix + "contact_points_description": [], # contact points description + "contact_points_group": [[0, 1, 2, 3], [4, 5, 6, 7]], + "contact_points_mask": [True, True], + "target_point_description": ["The center point on the bottom of the box."], + } + else: + data = { + "center": [0, 0, 0], + "extents": + half_size, + "scale": + half_size, + "target_pose": [[[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 1], [0, 0, 0, 1]]], + "contact_points_pose": [ + [[0, 0, 1, 0], [0, -1, 0, 0], [1, 0, 0, 0.7], [0, 0, 0, 1]], # front + [[0, -1, 0, 0], [0, 0, -1, 0], [1, 0, 0, 0.7], [0, 0, 0, 1]], # right + [[0, 1, 0, 0], [0, 0, 1, 0], [1, 0, 0, 0.7], [0, 0, 0, 1]], # left + [[0, 0, -1, 0], [0, 1, 0, 0], [1, 0, 0, 0.7], [0, 0, 0, 1]], # back + [[0, 0, 1, 0], [0, -1, 0, 0], [1, 0, 0, -0.7], [0, 0, 0, 1]], # front + [[0, -1, 0, 0], [0, 0, -1, 0], [1, 0, 0, -0.7], [0, 0, 0, 1]], # right + [[0, 1, 0, 0], [0, 0, 1, 0], [1, 0, 0, -0.7], [0, 0, 0, 1]], # left + [[0, 0, -1, 0], [0, 1, 0, 0], [1, 0, 0, -0.7], [0, 0, 0, 1]], # back + ], + "transform_matrix": + np.eye(4).tolist(), + "functional_matrix": [ + [ + [1.0, 0.0, 0.0, 0.0], + [0.0, -1.0, 0, 0.0], + [0.0, 0, -1.0, -1.0], + [0.0, 0.0, 0.0, 1.0], + ], + [ + [1.0, 0.0, 0.0, 0.0], + [0.0, -1.0, 0, 0.0], + [0.0, 0, -1.0, 1.0], + [0.0, 0.0, 0.0, 1.0], + ], + ], # functional points matrix + "contact_points_description": [], # contact points description + "contact_points_group": [[0, 1, 2, 3, 4, 5, 6, 7]], + "contact_points_mask": [True, True], + "target_point_description": ["The center point on the bottom of the box."], + } + return Actor(entity, data) + + +# create spere +def create_sphere( + scene, + pose: sapien.Pose, + radius: float, + color=None, + is_static=False, + name="", + texture_id=None, +) -> sapien.Entity: + scene, pose = preprocess(scene, pose) + entity = sapien.Entity() + entity.set_name(name) + entity.set_pose(pose) + + # create PhysX dynamic rigid body + rigid_component = (sapien.physx.PhysxRigidDynamicComponent() + if not is_static else sapien.physx.PhysxRigidStaticComponent()) + rigid_component.attach( + sapien.physx.PhysxCollisionShapeSphere(radius=radius, material=scene.default_physical_material)) + + # Add texture + if texture_id is not None: + + # test for both .png and .jpg + texturepath = f"./assets/textures/{texture_id}.png" + # create texture from file + texture2d = sapien.render.RenderTexture2D(texturepath) + material = sapien.render.RenderMaterial() + material.set_base_color_texture(texture2d) + # renderer.create_texture_from_file(texturepath) + # material.set_diffuse_texture(texturepath) + material.base_color = [1, 1, 1, 1] + material.metallic = 0.1 + material.roughness = 0.3 + else: + material = sapien.render.RenderMaterial(base_color=[*color[:3], 1]) + + # create render body for visualization + render_component = sapien.render.RenderBodyComponent() + render_component.attach( + # add a box visual shape with given size and rendering material + sapien.render.RenderShapeSphere(radius=radius, material=material)) + + entity.add_component(rigid_component) + entity.add_component(render_component) + entity.set_pose(pose) + + # in general, entity should only be added to scene after it is fully built + scene.add_entity(entity) + return entity + + +# create cylinder +def create_cylinder( + scene, + pose: sapien.Pose, + radius: float, + half_length: float, + color=None, + name="", +) -> sapien.Entity: + scene, pose = preprocess(scene, pose) + + entity = sapien.Entity() + entity.set_name(name) + entity.set_pose(pose) + + # create PhysX dynamic rigid body + rigid_component = sapien.physx.PhysxRigidDynamicComponent() + rigid_component.attach( + sapien.physx.PhysxCollisionShapeCylinder( + radius=radius, + half_length=half_length, + material=scene.default_physical_material, + )) + + # create render body for visualization + render_component = sapien.render.RenderBodyComponent() + render_component.attach( + # add a box visual shape with given size and rendering material + sapien.render.RenderShapeCylinder( + radius=radius, + half_length=half_length, + material=sapien.render.RenderMaterial(base_color=[*color[:3], 1]), + )) + + entity.add_component(rigid_component) + entity.add_component(render_component) + entity.set_pose(pose) + + # in general, entity should only be added to scene after it is fully built + scene.add_entity(entity) + return entity + + +# create box +def create_visual_box( + scene, + pose: sapien.Pose, + half_size, + color=None, + name="", +) -> sapien.Entity: + scene, pose = preprocess(scene, pose) + + entity = sapien.Entity() + entity.set_name(name) + entity.set_pose(pose) + + # create render body for visualization + render_component = sapien.render.RenderBodyComponent() + render_component.attach( + # add a box visual shape with given size and rendering material + sapien.render.RenderShapeBox(half_size, sapien.render.RenderMaterial(base_color=[*color[:3], 1]))) + + entity.add_component(render_component) + entity.set_pose(pose) + + # in general, entity should only be added to scene after it is fully built + scene.add_entity(entity) + return entity + + +def create_table( + scene, + pose: sapien.Pose, + length: float, + width: float, + height: float, + thickness=0.1, + color=(1, 1, 1), + name="table", + is_static=True, + texture_id=None, +) -> sapien.Entity: + """Create a table with specified dimensions.""" + scene, pose = preprocess(scene, pose) + builder = scene.create_actor_builder() + + if is_static: + builder.set_physx_body_type("static") + else: + builder.set_physx_body_type("dynamic") + + # Tabletop + tabletop_pose = sapien.Pose([0.0, 0.0, -thickness / 2]) # Center the tabletop at z=0 + tabletop_half_size = [length / 2, width / 2, thickness / 2] + builder.add_box_collision( + pose=tabletop_pose, + half_size=tabletop_half_size, + material=scene.default_physical_material, + ) + + # Add texture + if texture_id is not None: + + # test for both .png and .jpg + texturepath = f"./assets/background_texture/{texture_id}.png" + # create texture from file + texture2d = sapien.render.RenderTexture2D(texturepath) + material = sapien.render.RenderMaterial() + material.set_base_color_texture(texture2d) + # renderer.create_texture_from_file(texturepath) + # material.set_diffuse_texture(texturepath) + material.base_color = [1, 1, 1, 1] + material.metallic = 0.1 + material.roughness = 0.3 + builder.add_box_visual(pose=tabletop_pose, half_size=tabletop_half_size, material=material) + else: + builder.add_box_visual( + pose=tabletop_pose, + half_size=tabletop_half_size, + material=color, + ) + + # Table legs (x4) + leg_spacing = 0.1 + for i in [-1, 1]: + for j in [-1, 1]: + x = i * (length / 2 - leg_spacing / 2) + y = j * (width / 2 - leg_spacing / 2) + table_leg_pose = sapien.Pose([x, y, -height / 2 - 0.002]) + table_leg_half_size = [thickness / 2, thickness / 2, height / 2 - 0.002] + builder.add_box_collision(pose=table_leg_pose, half_size=table_leg_half_size) + builder.add_box_visual(pose=table_leg_pose, half_size=table_leg_half_size, material=color) + + builder.set_initial_pose(pose) + table = builder.build(name=name) + return table + + +# create obj model +def create_obj( + scene, + pose: sapien.Pose, + modelname: str, + scale=(1, 1, 1), + convex=False, + is_static=False, + model_id=None, + no_collision=False, +) -> Actor: + scene, pose = preprocess(scene, pose) + + modeldir = Path("assets/objects") / modelname + if model_id is None: + file_name = modeldir / "textured.obj" + json_file_path = modeldir / "model_data.json" + else: + file_name = modeldir / f"textured{model_id}.obj" + json_file_path = modeldir / f"model_data{model_id}.json" + + try: + with open(json_file_path, "r") as file: + model_data = json.load(file) + scale = model_data["scale"] + except: + model_data = None + + builder = scene.create_actor_builder() + if is_static: + builder.set_physx_body_type("static") + else: + builder.set_physx_body_type("dynamic") + + if not no_collision: + if convex == True: + builder.add_multiple_convex_collisions_from_file(filename=str(file_name), scale=scale) + else: + builder.add_nonconvex_collision_from_file(filename=str(file_name), scale=scale) + + builder.add_visual_from_file(filename=str(file_name), scale=scale) + mesh = builder.build(name=modelname) + mesh.set_pose(pose) + + return Actor(mesh, model_data) + + +# create glb model +def create_glb( + scene, + pose: sapien.Pose, + modelname: str, + scale=(1, 1, 1), + convex=False, + is_static=False, + model_id=None, +) -> Actor: + scene, pose = preprocess(scene, pose) + + modeldir = Path("./assets/objects") / modelname + if model_id is None: + file_name = modeldir / "base.glb" + json_file_path = modeldir / "model_data.json" + else: + file_name = modeldir / f"base{model_id}.glb" + json_file_path = modeldir / f"model_data{model_id}.json" + + try: + with open(json_file_path, "r") as file: + model_data = json.load(file) + scale = model_data["scale"] + except: + model_data = None + + builder = scene.create_actor_builder() + if is_static: + builder.set_physx_body_type("static") + else: + builder.set_physx_body_type("dynamic") + + if convex == True: + builder.add_multiple_convex_collisions_from_file(filename=str(file_name), scale=scale) + else: + builder.add_nonconvex_collision_from_file( + filename=str(file_name), + scale=scale, + ) + + builder.add_visual_from_file(filename=str(file_name), scale=scale) + mesh = builder.build(name=modelname) + mesh.set_pose(pose) + + return Actor(mesh, model_data) + + +def get_glb_or_obj_file(modeldir, model_id): + modeldir = Path(modeldir) + if model_id is None: + file = modeldir / "base.glb" + else: + file = modeldir / f"base{model_id}.glb" + if not file.exists(): + if model_id is None: + file = modeldir / "textured.obj" + else: + file = modeldir / f"textured{model_id}.obj" + return file + + +def create_actor( + scene, + pose: sapien.Pose, + modelname: str, + scale=(1, 1, 1), + convex=False, + is_static=False, + model_id=0, +) -> Actor: + scene, pose = preprocess(scene, pose) + modeldir = Path("assets/objects") / modelname + + if model_id is None: + json_file_path = modeldir / "model_data.json" + else: + json_file_path = modeldir / f"model_data{model_id}.json" + + collision_file = "" + visual_file = "" + if (modeldir / "collision").exists(): + collision_file = get_glb_or_obj_file(modeldir / "collision", model_id) + if collision_file == "" or not collision_file.exists(): + collision_file = get_glb_or_obj_file(modeldir, model_id) + + if (modeldir / "visual").exists(): + visual_file = get_glb_or_obj_file(modeldir / "visual", model_id) + if visual_file == "" or not visual_file.exists(): + visual_file = get_glb_or_obj_file(modeldir, model_id) + + if not collision_file.exists() or not visual_file.exists(): + print(modelname, "is not exist model file!") + return None + + try: + with open(json_file_path, "r") as file: + model_data = json.load(file) + scale = model_data["scale"] + except: + model_data = None + + builder = scene.create_actor_builder() + if is_static: + builder.set_physx_body_type("static") + else: + builder.set_physx_body_type("dynamic") + + if convex == True: + builder.add_multiple_convex_collisions_from_file(filename=str(collision_file), scale=scale) + else: + builder.add_nonconvex_collision_from_file( + filename=str(collision_file), + scale=scale, + ) + + builder.add_visual_from_file(filename=str(visual_file), scale=scale) + mesh = builder.build(name=modelname) + mesh.set_name(modelname) + mesh.set_pose(pose) + return Actor(mesh, model_data) + + +# create urdf model +def create_urdf_obj(scene, pose: sapien.Pose, modelname: str, scale=1.0, fix_root_link=True) -> ArticulationActor: + scene, pose = preprocess(scene, pose) + + modeldir = Path("./assets/objects") / modelname + json_file_path = modeldir / "model_data.json" + loader: sapien.URDFLoader = scene.create_urdf_loader() + loader.scale = scale + + try: + with open(json_file_path, "r") as file: + model_data = json.load(file) + loader.scale = model_data["scale"][0] + except: + model_data = None + + loader.fix_root_link = fix_root_link + loader.load_multiple_collisions_from_file = True + object: sapien.Articulation = loader.load(str(modeldir / "mobility.urdf")) + + object.set_root_pose(pose) + object.set_name(modelname) + return ArticulationActor(object, model_data) + + +def create_sapien_urdf_obj( + scene, + pose: sapien.Pose, + modelname: str, + scale=1.0, + modelid: int = None, + fix_root_link=False, +) -> ArticulationActor: + scene, pose = preprocess(scene, pose) + + modeldir = Path("assets") / "objects" / modelname + if modelid is not None: + model_list = [model for model in modeldir.iterdir() if model.is_dir() and model.name != "visual"] + + def extract_number(filename): + match = re.search(r"\d+", filename.name) + return int(match.group()) if match else 0 + + model_list = sorted(model_list, key=extract_number) + + if modelid >= len(model_list): + is_find = False + for model in model_list: + if modelid == int(model.name): + modeldir = model + is_find = True + break + if not is_find: + raise ValueError(f"modelid {modelid} is out of range for {modelname}.") + else: + modeldir = model_list[modelid] + json_file = modeldir / "model_data.json" + + if json_file.exists(): + with open(json_file, "r") as file: + model_data = json.load(file) + scale = model_data["scale"] + trans_mat = np.array(model_data.get("transform_matrix", np.eye(4))) + else: + model_data = None + trans_mat = np.eye(4) + + loader: sapien.URDFLoader = scene.create_urdf_loader() + loader.scale = scale + loader.fix_root_link = fix_root_link + loader.load_multiple_collisions_from_file = True + object = loader.load_multiple(str(modeldir / "mobility.urdf"))[0][0] + + pose_mat = pose.to_transformation_matrix() + pose = sapien.Pose( + p=pose_mat[:3, 3] + trans_mat[:3, 3], + q=t3d.quaternions.mat2quat(trans_mat[:3, :3] @ pose_mat[:3, :3]), + ) + object.set_pose(pose) + + if model_data is not None: + if "init_qpos" in model_data and len(model_data["init_qpos"]) > 0: + object.set_qpos(np.array(model_data["init_qpos"])) + if "mass" in model_data and len(model_data["mass"]) > 0: + for link in object.get_links(): + link.set_mass(model_data["mass"].get(link.get_name(), 0.1)) + + bounding_box_file = modeldir / "bounding_box.json" + if bounding_box_file.exists(): + bounding_box = json.load(open(bounding_box_file, "r", encoding="utf-8")) + model_data["extents"] = (np.array(bounding_box["max"]) - np.array(bounding_box["min"])).tolist() + object.set_name(modelname) + return ArticulationActor(object, model_data) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/get_camera_config.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/get_camera_config.py new file mode 100644 index 00000000..af6f7aa1 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/get_camera_config.py @@ -0,0 +1,14 @@ +import yaml, os +from envs._GLOBAL_CONFIGS import CONFIGS_PATH + + +def get_camera_config(camera_type): + camera_config_path = os.path.join(CONFIGS_PATH, "_camera_config.yml") + + assert os.path.isfile(camera_config_path), "task config file is missing" + + with open(camera_config_path, "r", encoding="utf-8") as f: + camera_args = yaml.load(f.read(), Loader=yaml.FullLoader) + + assert camera_type in camera_args, f"camera {camera_type} is not defined" + return camera_args[camera_type] diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/images_to_video.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/images_to_video.py new file mode 100644 index 00000000..24644ada --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/images_to_video.py @@ -0,0 +1,51 @@ +import cv2 +import numpy as np +import os +import subprocess +import pickle +import pdb + + +def images_to_video(imgs: np.ndarray, out_path: str, fps: float = 30.0, is_rgb: bool = True) -> None: + if (not isinstance(imgs, np.ndarray) or imgs.ndim != 4 or imgs.shape[3] not in (3, 4)): + raise ValueError("imgs must be a numpy.ndarray of shape (N, H, W, C), with C equal to 3 or 4.") + os.makedirs(os.path.dirname(out_path), exist_ok=True) + n_frames, H, W, C = imgs.shape + if C == 3: + pixel_format = "rgb24" if is_rgb else "bgr24" + else: + pixel_format = "rgba" + ffmpeg = subprocess.Popen( + [ + "ffmpeg", + "-y", + "-loglevel", + "error", + "-f", + "rawvideo", + "-pixel_format", + pixel_format, + "-video_size", + f"{W}x{H}", + "-framerate", + str(fps), + "-i", + "-", + "-pix_fmt", + "yuv420p", + "-vcodec", + "libx264", + "-crf", + "23", + f"{out_path}", + ], + stdin=subprocess.PIPE, + ) + ffmpeg.stdin.write(imgs.tobytes()) + ffmpeg.stdin.close() + if ffmpeg.wait() != 0: + raise IOError(f"Cannot open ffmpeg. Please check the output path and ensure ffmpeg is supported.") + + print( + f"🎬 Video is saved to `{out_path}`, containing \033[94m{n_frames}\033[0m frames at {W}×{H} resolution and {fps} FPS." + ) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/parse_hdf5.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/parse_hdf5.py new file mode 100644 index 00000000..95759190 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/parse_hdf5.py @@ -0,0 +1,58 @@ +import h5py, cv2 +import numpy as np + + +def parse_img_array(data): + """ + 将一个字节流数组解码为图像数组。 + + Args: + data: np.ndarray of shape (N,), 每个元素要么是 Python bytes,要么是 np.ndarray(dtype=uint8) + Returns: + imgs: np.ndarray of shape (N, H, W, C), dtype=uint8 + """ + # 确保 data 是可迭代的一维数组 + flat = data.ravel() + + imgs = [] + for buf in flat: + # buf 可能是 bytes,也可能是 np.ndarray(dtype=uint8) + if isinstance(buf, (bytes, bytearray)): + arr = np.frombuffer(buf, dtype=np.uint8) + elif isinstance(buf, np.ndarray) and buf.dtype == np.uint8: + arr = buf + else: + raise TypeError(f"Unsupported buffer type: {type(buf)}") + + # 解码成 BGR 图像 + img = cv2.imdecode(arr, cv2.IMREAD_COLOR) + if img is None: + raise ValueError("cv2.imdecode 返回 None,说明字节流可能不是有效的图片格式") + imgs.append(img) + + # 将 list 转成形如 (N, H, W, C) 的 ndarray + return np.stack(imgs, axis=0) + + +def h5_to_dict(node): + result = {} + for name, item in node.items(): + if isinstance(item, h5py.Dataset): + data = item[()] + if "rgb" in name: + result[name] = parse_img_array(data) + else: + result[name] = data + elif isinstance(item, h5py.Group): + # 递归处理子 group + result[name] = h5_to_dict(item) + # 如果你还想把 attributes 一并读进来,可以: + if hasattr(node, "attrs") and len(node.attrs) > 0: + result["_attrs"] = dict(node.attrs) + return result + + +def read_hdf5(file_path): + with h5py.File(file_path, "r") as f: + data_dict = h5_to_dict(f) + return data_dict diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/pkl2hdf5.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/pkl2hdf5.py new file mode 100644 index 00000000..267f421c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/pkl2hdf5.py @@ -0,0 +1,109 @@ +import h5py, pickle +import numpy as np +import os +import cv2 +from collections.abc import Mapping, Sequence +import shutil +from .images_to_video import images_to_video + + +def images_encoding(imgs): + encode_data = [] + padded_data = [] + max_len = 0 + for i in range(len(imgs)): + success, encoded_image = cv2.imencode(".jpg", imgs[i]) + jpeg_data = encoded_image.tobytes() + encode_data.append(jpeg_data) + max_len = max(max_len, len(jpeg_data)) + # padding + for i in range(len(imgs)): + padded_data.append(encode_data[i].ljust(max_len, b"\0")) + return encode_data, max_len + + +def parse_dict_structure(data): + if isinstance(data, dict): + parsed = {} + for key, value in data.items(): + if isinstance(value, dict): + parsed[key] = parse_dict_structure(value) + elif isinstance(value, np.ndarray): + parsed[key] = [] + else: + parsed[key] = [] + return parsed + else: + return [] + + +def append_data_to_structure(data_structure, data): + for key in data_structure: + if key in data: + if isinstance(data_structure[key], list): + # 如果是叶子节点,直接追加数据 + data_structure[key].append(data[key]) + elif isinstance(data_structure[key], dict): + # 如果是嵌套字典,递归处理 + append_data_to_structure(data_structure[key], data[key]) + + +def load_pkl_file(pkl_path): + with open(pkl_path, "rb") as f: + data = pickle.load(f) + return data + + +def create_hdf5_from_dict(hdf5_group, data_dict): + for key, value in data_dict.items(): + if isinstance(value, dict): + subgroup = hdf5_group.create_group(key) + create_hdf5_from_dict(subgroup, value) + elif isinstance(value, list): + value = np.array(value) + if "rgb" in key: + encode_data, max_len = images_encoding(value) + hdf5_group.create_dataset(key, data=encode_data, dtype=f"S{max_len}") + else: + hdf5_group.create_dataset(key, data=value) + else: + return + try: + hdf5_group.create_dataset(key, data=str(value)) + print("Not np array") + except Exception as e: + print(f"Error storing value for key '{key}': {e}") + + +def pkl_files_to_hdf5_and_video(pkl_files, hdf5_path, video_path): + data_list = parse_dict_structure(load_pkl_file(pkl_files[0])) + for pkl_file_path in pkl_files: + pkl_file = load_pkl_file(pkl_file_path) + append_data_to_structure(data_list, pkl_file) + + images_to_video(np.array(data_list["observation"]["head_camera"]["rgb"]), out_path=video_path) + + with h5py.File(hdf5_path, "w") as f: + create_hdf5_from_dict(f, data_list) + + +def process_folder_to_hdf5_video(folder_path, hdf5_path, video_path): + pkl_files = [] + for fname in os.listdir(folder_path): + if fname.endswith(".pkl") and fname[:-4].isdigit(): + pkl_files.append((int(fname[:-4]), os.path.join(folder_path, fname))) + + if not pkl_files: + raise FileNotFoundError(f"No valid .pkl files found in {folder_path}") + + pkl_files.sort() + pkl_files = [f[1] for f in pkl_files] + + expected = 0 + for f in pkl_files: + num = int(os.path.basename(f)[:-4]) + if num != expected: + raise ValueError(f"Missing file {expected}.pkl") + expected += 1 + + pkl_files_to_hdf5_and_video(pkl_files, hdf5_path, video_path) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/rand_create_actor.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/rand_create_actor.py new file mode 100644 index 00000000..de1e5cce --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/rand_create_actor.py @@ -0,0 +1,217 @@ +import sapien.core as sapien +import numpy as np +import transforms3d as t3d +import sapien.physx as sapienp +from .create_actor import * + + +def rand_pose( + xlim: np.ndarray, + ylim: np.ndarray, + zlim: np.ndarray = [0.741], + ylim_prop=False, + rotate_rand=False, + rotate_lim=[0, 0, 0], + qpos=[1, 0, 0, 0], +) -> sapien.Pose: + if len(xlim) < 2 or xlim[1] < xlim[0]: + xlim = np.array([xlim[0], xlim[0]]) + if len(ylim) < 2 or ylim[1] < ylim[0]: + ylim = np.array([ylim[0], ylim[0]]) + if len(zlim) < 2 or zlim[1] < zlim[0]: + zlim = np.array([zlim[0], zlim[0]]) + + x = np.random.uniform(xlim[0], xlim[1]) + y = np.random.uniform(ylim[0], ylim[1]) + + while ylim_prop and abs(x) < 0.15 and y > 0: + y = np.random.uniform(ylim[0], 0) + + z = np.random.uniform(zlim[0], zlim[1]) + + rotate = qpos + if rotate_rand: + angles = [0, 0, 0] + for i in range(3): + angles[i] = np.random.uniform(-rotate_lim[i], rotate_lim[i]) + rotate_quat = t3d.euler.euler2quat(angles[0], angles[1], angles[2]) + rotate = t3d.quaternions.qmult(rotate, rotate_quat) + + return sapien.Pose([x, y, z], rotate) + + +def rand_create_obj( + scene, + modelname: str, + xlim: np.ndarray, + ylim: np.ndarray, + zlim: np.ndarray = [0.741], + ylim_prop=False, + rotate_rand=False, + rotate_lim=[0, 0, 0], + qpos=[1, 0, 0, 0], + scale=(1, 1, 1), + convex=False, + is_static=False, + model_id=None, +) -> Actor: + + obj_pose = rand_pose( + xlim=xlim, + ylim=ylim, + zlim=zlim, + ylim_prop=ylim_prop, + rotate_rand=rotate_rand, + rotate_lim=rotate_lim, + qpos=qpos, + ) + + return create_obj( + scene=scene, + pose=obj_pose, + modelname=modelname, + scale=scale, + convex=convex, + is_static=is_static, + model_id=model_id, + ) + + +def rand_create_glb( + scene, + modelname: str, + xlim: np.ndarray, + ylim: np.ndarray, + zlim: np.ndarray = [0.741], + ylim_prop=False, + rotate_rand=False, + rotate_lim=[0, 0, 0], + qpos=[1, 0, 0, 0], + scale=(1, 1, 1), + convex=False, + is_static=False, + model_id=None, +) -> Actor: + + obj_pose = rand_pose( + xlim=xlim, + ylim=ylim, + zlim=zlim, + ylim_prop=ylim_prop, + rotate_rand=rotate_rand, + rotate_lim=rotate_lim, + qpos=qpos, + ) + + return create_glb( + scene=scene, + pose=obj_pose, + modelname=modelname, + scale=scale, + convex=convex, + is_static=is_static, + model_id=model_id, + ) + + +def rand_create_urdf_obj( + scene, + modelname: str, + xlim: np.ndarray, + ylim: np.ndarray, + zlim: np.ndarray = [0.741], + ylim_prop=False, + rotate_rand=False, + rotate_lim=[0, 0, 0], + qpos=[1, 0, 0, 0], + scale=1.0, + fix_root_link=True, +) -> ArticulationActor: + + obj_pose = rand_pose( + xlim=xlim, + ylim=ylim, + zlim=zlim, + ylim_prop=ylim_prop, + rotate_rand=rotate_rand, + rotate_lim=rotate_lim, + qpos=qpos, + ) + + return create_urdf_obj( + scene, + pose=obj_pose, + modelname=modelname, + scale=scale, + fix_root_link=fix_root_link, + ) + + +def rand_create_sapien_urdf_obj( + scene, + modelname: str, + modelid: int, + xlim: np.ndarray, + ylim: np.ndarray, + zlim: np.ndarray = [0.741], + ylim_prop=False, + rotate_rand=False, + rotate_lim=[0, 0, 0], + qpos=[1, 0, 0, 0], + scale=1.0, + fix_root_link=False, +) -> ArticulationActor: + obj_pose = rand_pose( + xlim=xlim, + ylim=ylim, + zlim=zlim, + ylim_prop=ylim_prop, + rotate_rand=rotate_rand, + rotate_lim=rotate_lim, + qpos=qpos, + ) + return create_sapien_urdf_obj( + scene=scene, + pose=obj_pose, + modelname=modelname, + modelid=modelid, + scale=scale, + fix_root_link=fix_root_link, + ) + + +def rand_create_actor( + scene, + modelname: str, + xlim: np.ndarray, + ylim: np.ndarray, + zlim: np.ndarray = [0.741], + ylim_prop=False, + rotate_rand=False, + rotate_lim=[0, 0, 0], + qpos=[1, 0, 0, 0], + scale=(1, 1, 1), + convex=False, + is_static=False, + model_id=0, +) -> Actor: + + obj_pose = rand_pose( + xlim=xlim, + ylim=ylim, + zlim=zlim, + ylim_prop=ylim_prop, + rotate_rand=rotate_rand, + rotate_lim=rotate_lim, + qpos=qpos, + ) + + return create_actor( + scene=scene, + pose=obj_pose, + modelname=modelname, + scale=scale, + convex=convex, + is_static=is_static, + model_id=model_id, + ) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/rand_create_cluttered_actor.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/rand_create_cluttered_actor.py new file mode 100644 index 00000000..15cd18dc --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/rand_create_cluttered_actor.py @@ -0,0 +1,279 @@ +import sapien.core as sapien +import numpy as np +import transforms3d as t3d +import sapien.physx as sapienp +from .create_actor import * + +import re +import json +from pathlib import Path + + +def get_all_cluttered_objects(): + cluttered_objects_info = {} + cluttered_objects_name = [] + + # load from cluttered_objects + cluttered_objects_config = json.load(open(Path("./assets/objects/objaverse/list.json"), "r", encoding="utf-8")) + cluttered_objects_name += cluttered_objects_config["item_names"] + for model_name, model_ids in cluttered_objects_config["list_of_items"].items(): + cluttered_objects_info[model_name] = { + "ids": model_ids, + "type": "urdf", + "root": f"objects/objaverse/{model_name}", + } + params = {} + for model_id in model_ids: + model_full_name = f"{model_name}_{model_id}" + params[model_id] = { + "z_max": cluttered_objects_config["z_max"][model_full_name], + "radius": cluttered_objects_config["radius"][model_full_name], + "z_offset": cluttered_objects_config["z_offset"][model_full_name], + } + cluttered_objects_info[model_name]["params"] = params + + # load from objects + objects_dir = Path("./assets/objects") + for model_dir in objects_dir.iterdir(): + if not model_dir.is_dir(): + continue + if re.search(r"^(\d+)_(.*)", model_dir.name) is None: + continue + model_name = model_dir.name + model_id_list, params = [], {} + for model_cfg in model_dir.iterdir(): + if model_cfg.is_dir() or model_cfg.suffix != ".json": + continue + + # get model id + model_id = re.search(r"model_data(\d+)", model_cfg.name) + if not model_id: + continue + model_id = model_id.group(1) + + try: + # get model params + model_config: dict = json.load(open(model_cfg, "r", encoding="utf-8")) + if "center" not in model_config or "extents" not in model_config: + continue + if model_config.get("stable", False) is False: + continue + center = model_config["center"] + extents = model_config["extents"] + scale = model_config.get("scale", [1.0, 1.0, 1.0]) + # 0: x, 1: z, 2: y + params[model_id] = { + "z_max": (extents[1] + center[1]) * scale[1], + "radius": max(extents[0] * scale[0], extents[2] * scale[2]) / 2, + "z_offset": 0, + } + model_id_list.append(model_id) + except Exception as e: + print(f"Error loading model config {model_cfg}: {e}") + if len(model_id_list) == 0: + continue + cluttered_objects_name.append(model_name) + model_id_list.sort() + cluttered_objects_info[model_name] = { + "ids": model_id_list, + "type": "glb", + "root": f"objects/{model_name}", + "params": params, + } + + same_obj = json.load(open(Path("./assets/objects/same.json"), "r", encoding="utf-8")) + cluttered_objects_name = list(cluttered_objects_name) + cluttered_objects_name.sort() + return cluttered_objects_info, cluttered_objects_name, same_obj + + +cluttered_objects_info, cluttered_objects_list, same_obj = get_all_cluttered_objects() + + +def get_available_cluttered_objects(entity_on_scene: list): + global cluttered_objects_info, cluttered_objects_list, same_obj + + model_in_use = [] + for entity_name in entity_on_scene: + if same_obj.get(entity_name) is not None: + model_in_use += same_obj[entity_name] + model_in_use.append(entity_name) + + available_models = set(cluttered_objects_list) - set(model_in_use) + available_models = list(available_models) + available_models.sort() + return available_models, cluttered_objects_info + + +def check_overlap(radius, x, y, area): + if x <= area[0]: + dx = area[0] - x + elif area[0] < x and x < area[2]: + dx = 0 + elif x >= area[2]: + dx = x - area[2] + if y <= area[1]: + dy = area[1] - y + elif area[1] < y and y < area[3]: + dy = 0 + elif y >= area[3]: + dy = y - area[3] + + return dx * dx + dy * dy <= radius * radius + + +def rand_pose_cluttered( + xlim: np.ndarray, + ylim: np.ndarray, + zlim: np.ndarray, + ylim_prop=False, + rotate_rand=False, + rotate_lim=[0, 0, 0], + qpos=[1, 0, 0, 0], + size_dict=None, + obj_radius=0.1, + z_offset=0.001, + z_max=0, + prohibited_area=None, + obj_margin=0.005, +) -> sapien.Pose: + if len(xlim) < 2 or xlim[1] < xlim[0]: + xlim = np.array([xlim[0], xlim[0]]) + if len(ylim) < 2 or ylim[1] < ylim[0]: + ylim = np.array([ylim[0], ylim[0]]) + if len(zlim) < 2 or zlim[1] < zlim[0]: + zlim = np.array([zlim[0], zlim[0]]) + + times = 0 + while True: + times += 1 + if times > 100: + return False, None + x = np.random.uniform(xlim[0], xlim[1]) + y = np.random.uniform(ylim[0], ylim[1]) + new_obj_radius = obj_radius + obj_margin + is_overlap = False + for area in prohibited_area: + if check_overlap(new_obj_radius, x, y, area): + is_overlap = True + break + if is_overlap: + continue + distances = np.sqrt((np.array([sub_list[0] for sub_list in size_dict]) - x)**2 + + (np.array([sub_list[1] for sub_list in size_dict]) - y)**2) + max_distances = np.array([sub_list[3] + new_obj_radius + obj_margin for sub_list in size_dict]) + + if y - new_obj_radius < 0: + if z_max > 0.05: + continue + if (x - new_obj_radius < -0.6 or x + new_obj_radius > 0.6 or y - new_obj_radius < -0.34 + or y + new_obj_radius > 0.34): + continue + if np.all(distances > max_distances) and y + new_obj_radius < ylim[1]: + break + + z = np.random.uniform(zlim[0], zlim[1]) + z = z - z_offset + + rotate = qpos + if rotate_rand: + angles = [0, 0, 0] + for i in range(3): + angles[i] = np.random.uniform(-rotate_lim[i], rotate_lim[i]) + rotate_quat = t3d.euler.euler2quat(angles[0], angles[1], angles[2]) + rotate = t3d.quaternions.qmult(rotate, rotate_quat) + + return True, sapien.Pose([x, y, z], rotate) + + +def rand_create_cluttered_actor( + scene, + modelname: str, + modelid: str, + modeltype: str, + xlim: np.ndarray, + ylim: np.ndarray, + zlim: np.ndarray, + ylim_prop=False, + rotate_rand=False, + rotate_lim=[0, 0, 0], + qpos=None, + scale=(1, 1, 1), + convex=True, + is_static=False, + size_dict=None, + obj_radius=0.1, + z_offset=0.001, + z_max=0, + fix_root_link=True, + prohibited_area=None, +) -> tuple[bool, Actor | None]: + + if qpos is None: + if modeltype == "glb": + qpos = [0.707107, 0.707107, 0, 0] + rotate_lim = [rotate_lim[0], rotate_lim[2], rotate_lim[1]] + else: + qpos = [1, 0, 0, 0] + + success, obj_pose = rand_pose_cluttered( + xlim=xlim, + ylim=ylim, + zlim=zlim, + ylim_prop=ylim_prop, + rotate_rand=rotate_rand, + rotate_lim=rotate_lim, + qpos=qpos, + size_dict=size_dict, + obj_radius=obj_radius, + z_offset=z_offset, + z_max=z_max, + prohibited_area=prohibited_area, + ) + + if not success: + return False, None + + if modeltype == "urdf": + obj = create_cluttered_urdf_obj( + scene=scene, + pose=obj_pose, + modelname=f"objects/objaverse/{modelname}/{modelid}", + scale=scale if isinstance(scale, float) else scale[0], + fix_root_link=fix_root_link, + ) + if obj is None: + return False, None + else: + return True, obj + else: + obj = create_actor( + scene=scene, + pose=obj_pose, + modelname=modelname, + model_id=modelid, + scale=scale, + convex=convex, + is_static=is_static, + ) + if obj is None: + return False, None + else: + return True, obj + + +def create_cluttered_urdf_obj(scene, pose: sapien.Pose, modelname: str, scale=1.0, fix_root_link=True) -> Actor: + scene, pose = preprocess(scene, pose) + modeldir = Path("assets") / modelname + + loader: sapien.URDFLoader = scene.create_urdf_loader() + loader.scale = scale + loader.fix_root_link = fix_root_link + loader.load_multiple_collisions_from_file = False + object: sapien.Articulation = loader.load_multiple(str(modeldir / "model.urdf"))[1][0] + object.set_pose(pose) + + if isinstance(object, sapien.physx.PhysxArticulation): + return ArticulationActor(object, None) + else: + return Actor(object, None) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/save_file.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/save_file.py new file mode 100644 index 00000000..53b3cda7 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/save_file.py @@ -0,0 +1,44 @@ +import numpy as np +from PIL import Image, ImageColor + +import json + +import os +import pickle +import open3d as o3d + + +def ensure_dir(file_path): + directory = os.path.dirname(file_path) + if not os.path.exists(directory): + os.makedirs(directory) + + +def save_img(save_path, img_file): + img = Image.fromarray(img_file) + ensure_dir(save_path) + img.save(save_path) + + +def save_json(save_path, json_file): + ensure_dir(save_path) + with open(save_path, "w") as f: + json.dump(json_file, f, indent=4) + + +def save_pkl(save_path, dic_file): + ensure_dir(save_path) + with open(save_path, "wb") as f: + pickle.dump(dic_file, f) + + +def save_pcd(save_path, pcd_arr, color=False): + ensure_dir(save_path) + point_cloud = o3d.geometry.PointCloud() + point_arr = pcd_arr[:, :3] + point_cloud.points = o3d.utility.Vector3dVector(point_arr) + if color: + colors_arr = pcd_arr[:, -3:] + point_cloud.colors = o3d.utility.Vector3dVector(colors_arr) + + o3d.io.write_point_cloud(save_path, point_cloud) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/transforms.py b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/transforms.py new file mode 100644 index 00000000..a8994404 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/envs/utils/transforms.py @@ -0,0 +1,530 @@ +import numpy as np +from pathlib import Path +import sapien.core as sapien +import transforms3d as t3d +from typing import Literal + + +def pause(task, till_close=False, show_point=False): + if show_point: + for point in Point.points: + point.update() + task.viewer.paused = True + while task.viewer.paused: + task.viewer.render() + if till_close: + while not task.viewer.closed: + for point in Point.points: + point.update() + task.scene.step() + task.scene.update_render() + task.viewer.render() + + +import time +from functools import wraps + + +def timer(func): + + @wraps(func) + def decorated(*args, **kwargs): + name = func.__name__ + start_time = time.perf_counter() + ret = func(*args, **kwargs) + elapsed = time.perf_counter() - start_time + print(f"Timer '{name}': {elapsed:.4f} seconds") + with open("timer.log", "a", encoding="utf-8") as f: + f.write(f"Timer '{name}': {elapsed:.4f} seconds\n") + return ret + + return decorated + + +timer_dict = {} + + +def local_timer(name: str): + if name in timer_dict: + elapsed = time.perf_counter() - timer_dict[name] + print(f"Local Timer '{name}': {elapsed:.4f} seconds") + with open("timer.log", "a", encoding="utf-8") as f: + f.write(f"Local Timer '{name}': {elapsed:.4f} seconds\n") + del timer_dict[name] + else: + timer_dict[name] = time.perf_counter() + + +class Point: + points: list["Point"] = [] + """特定 base 坐标系下的点""" + + def __init__( + self, + scene: sapien.Scene, + base: sapien.Entity, + base_scale: float, + init_mat: np.ndarray, + base_pose_mat: np.ndarray = None, + scaled: bool = True, + follow: sapien.Entity = None, + name: str = "point", + size: float = 0.05, + eular_round_to: int = 0.01, + ): + self.name = name + self.scene = scene + self.base = base + if base_pose_mat is not None: + self.base_pose_mat = np.array(base_pose_mat) + else: + self.base_pose_mat = base.get_pose().to_transformation_matrix() + self.follow = follow + self.base_scale = base_scale + self.eular_round_to = eular_round_to + + self.mat = np.array(init_mat) + if not scaled: + self.mat[:3, 3] *= self.base_scale + + self.pose = self.trans_base( + self.base.get_pose().to_transformation_matrix(), + self.base_pose_mat, + self.mat, + ) + self.mat = self.word2base(self.pose.to_transformation_matrix()).to_transformation_matrix() + self.base_pose_mat = self.base.get_pose().to_transformation_matrix() + + builder = scene.create_actor_builder() + builder.set_physx_body_type("static") + builder.add_visual_from_file(filename="./assets/objects/cube/textured.obj", scale=[size, size, size]) + self.point = builder.build(name=name) + self.point.set_pose(self.pose) + Point.points.append(self) + + def __del__(self): + Point.points.remove(self) + + def get_pose(self) -> sapien.Pose: + return self.pose + + @staticmethod + def pose2list(pose: sapien.Pose) -> list: + return pose.p.tolist() + pose.q.tolist() + + @staticmethod + def round_eular(eular, round_to: int = 1) -> np.ndarray: + unit = round_to / 180 * np.pi + return np.round(np.array(eular) / unit) * unit + + @staticmethod + def trans_mat(to_mat: np.ndarray, from_mat: np.ndarray, scale: float = 1.0): + to_rot = to_mat[:3, :3] + from_rot = from_mat[:3, :3] + rot_mat = to_rot @ from_rot.T + + trans_mat = (to_mat[:3, 3] - from_mat[:3, 3]) / scale + + result = np.eye(4) + result[:3, :3] = rot_mat + result[:3, 3] = trans_mat + result = np.where(np.abs(result) < 1e-5, 0, result) + return result + + @staticmethod + def trans_pose(to_pose: sapien.Pose, from_pose: sapien.Pose, scale: float = 1.0): + return Point.trans_mat( + to_pose.to_transformation_matrix(), + from_pose.to_transformation_matrix(), + scale, + ) + + @staticmethod + def trans_base( + now_base_mat: np.ndarray, + init_base_mat: np.ndarray, + init_pose_mat: np.ndarray, + scale: float = 1.0, + ): + now_base_mat = np.array(now_base_mat) + init_base_mat = np.array(init_base_mat) + init_pose_mat = np.array(init_pose_mat) + init_pose_mat[:3, 3] *= scale + + now_pose_mat = np.eye(4) + base_trans_mat = Point.trans_mat(now_base_mat, init_base_mat) + now_pose_mat[:3, :3] = (base_trans_mat[:3, :3] @ init_pose_mat[:3, :3] @ base_trans_mat[:3, :3].T) + now_pose_mat[:3, 3] = base_trans_mat[:3, :3] @ init_pose_mat[:3, 3] + + # 转化为世界坐标 + p = now_pose_mat[:3, 3] + now_base_mat[:3, 3] + q_mat = now_pose_mat[:3, :3] @ now_base_mat[:3, :3] + return sapien.Pose(p, t3d.quaternions.mat2quat(q_mat)) + + def get_output_mat(self): + opt_mat = self.mat.copy() + opt_mat[:3, 3] /= self.base_scale + return opt_mat + + def base2world(self, entity_mat, scale=1.0) -> sapien.Pose: + """将 base 坐标系下的矩阵转换到世界坐标系下""" + entity_mat = np.array(entity_mat) + base_mat = self.base.get_pose().to_transformation_matrix() + p = entity_mat[:3, 3] * scale + base_mat[:3, 3] + q_mat = entity_mat[:3, :3] @ base_mat[:3, :3] + return sapien.Pose(p, t3d.quaternions.mat2quat(q_mat)) + + def word2base(self, entity_mat, scale=1.0) -> sapien.Pose: + """将世界坐标系下的矩阵转换到 base 坐标系下""" + entity_mat = np.array(entity_mat) + base_mat = self.base.get_pose().to_transformation_matrix() + p = entity_mat[:3, 3] - base_mat[:3, 3] + q_mat = entity_mat[:3, :3] @ base_mat[:3, :3].T + return sapien.Pose(p, t3d.quaternions.mat2quat(q_mat)) + + def set_pose(self, new_pose: sapien.Pose): + """更新点的位置""" + self.pose = new_pose + self.point.set_pose(self.pose) + self.mat = self.word2base(new_pose.to_transformation_matrix()).to_transformation_matrix() + + print("set", self.name) + print(self.get_output_mat().tolist()) + print() + + def update(self, force_output: bool = False, flexible: bool = False): + new_mat = np.eye(4) + if self.follow is not None: + new_mat = self.trans_mat( + self.follow.get_pose().to_transformation_matrix(), + self.base.get_pose().to_transformation_matrix(), + ) + elif flexible: + new_mat = self.trans_mat( + self.point.get_pose().to_transformation_matrix(), + self.base.get_pose().to_transformation_matrix(), + ) + else: + new_mat = self.word2base( + self.trans_base( + self.base.get_pose().to_transformation_matrix(), + self.base_pose_mat, + self.mat, + ).to_transformation_matrix()).to_transformation_matrix() + + new_mat[:3, :3] = t3d.euler.euler2mat( + *self.round_eular(t3d.euler.mat2euler(new_mat[:3, :3]), self.eular_round_to)) + self.pose = self.base2world(new_mat) + self.point.set_pose(self.pose) + + if not np.allclose(new_mat, self.mat, atol=1e-3) or force_output: + self.mat = new_mat + self.base_pose_mat = self.base.get_pose().to_transformation_matrix() + print("update", self.name) + if self.name == "left": + print("'lb': ", self.get_output_mat().tolist(), ", ", sep="") + elif self.name == "right": + print("'rb': ", self.get_output_mat().tolist(), ", ", sep="") + else: + print(self.get_output_mat().tolist()) + print( + "init_base_mat =", + self.base.get_pose().to_transformation_matrix().tolist(), + ) + print() + + +def rotate_cone(new_pt: np.ndarray, origin: np.ndarray, z_dir: np.ndarray = [0, 0, 1]): + x = origin - new_pt + x = x / np.linalg.norm(x) + bx_ = np.array(z_dir).reshape(3) + z = bx_ - np.dot(x, bx_) * x + z = z / np.linalg.norm(z) + y = np.cross(z, x) + return np.stack([x, y, z], axis=1) + + +def _tolist(pose: sapien.Pose | list | np.ndarray) -> list: + if isinstance(pose, list): + return pose + elif isinstance(pose, sapien.Pose): + return pose.p.tolist() + pose.q.tolist() + else: + return pose.tolist() + + +def _toPose(pose: sapien.Pose | list | np.ndarray) -> sapien.Pose: + if isinstance(pose, list): + assert len(pose) == 7 or len(pose) == 3 + if len(pose) == 3: + return sapien.Pose(pose[:3], [1, 0, 0, 0]) + else: + return sapien.Pose(pose[:3], pose[3:]) + elif isinstance(pose, np.ndarray): + assert pose.shape == (7, ) or pose.shape == (3, ) + if pose.shape == (3, ): + return sapien.Pose(pose[:3], [1, 0, 0, 0]) + else: + return sapien.Pose(pose[:3], pose[3:]) + else: + return pose + + +def rotate_along_axis( + target_pose, + center_pose, + axis, + theta: float = np.pi / 2, + axis_type: Literal["center", "target", "world"] = "center", + towards=None, + camera_face=None, +) -> list: + """ + 以 center 为中心,沿着指定轴旋转指定角度。通过 towards 可指定旋转方向(方向为 center->target 向量与 towards 向量乘积为正的方向) + + target_pose: 目标点(比如在物体正上方的预抓取点) + center_pose: 中心点(比如物体的位置) + axis: 旋转轴 + theta: 旋转角度(单位:弧度) + axis_type: 旋转轴的类型('center':相对于 center_pose,'target':相对于 target_pose,'world':世界坐标系),默认是 'center' + towards: 旋转方向(可选),如果指定了这个参数,则会根据这个参数来决定旋转的方向 + camera_face: 相机朝向(可选),会限制相机向量与该向量点积为正;如果设置为 None,只旋转不考虑相机朝向 + 返回值:列表,前3个元素是坐标,后4个元素是四元数 + """ + target_pose, center_pose = _toPose(target_pose), _toPose(center_pose) + if theta == 0: + return target_pose.p.tolist() + target_pose.q.tolist() + rotate_mat = t3d.axangles.axangle2mat(axis, theta) + + target_mat = target_pose.to_transformation_matrix() + center_mat = center_pose.to_transformation_matrix() + if axis_type == "center": + world_axis = (center_mat[:3, :3] @ np.array(axis).reshape(3, 1)).reshape(3) + elif axis_type == "target": + world_axis = (target_mat[:3, :3] @ np.array(axis).reshape(3, 1)).reshape(3) + else: + world_axis = np.array(axis).reshape(3) + + rotate_mat = t3d.axangles.axangle2mat(world_axis, theta) + p = (rotate_mat @ (target_pose.p - center_pose.p).reshape(3, 1)).reshape(3) + center_pose.p + if towards is not None: + towards = np.dot(p - center_pose.p, np.array(towards).reshape(3)) + if towards < 0: + rotate_mat = t3d.axangles.axangle2mat(world_axis, -theta) + p = (rotate_mat @ (target_pose.p - center_pose.p).reshape(3, 1)).reshape(3) + center_pose.p + + if camera_face is None: + q = t3d.quaternions.mat2quat(rotate_mat @ target_mat[:3, :3]) + else: + q = t3d.quaternions.mat2quat(rotate_cone(p, center_pose.p, camera_face)) + return p.tolist() + q.tolist() + + +def rotate2rob(target_pose, rob_pose, box_pose, theta: float = 0.5) -> list: + """ + 向指定的 rob_pose 偏移 + """ + target_pose, rob_pose, box_pose = ( + _toPose(target_pose), + _toPose(rob_pose), + _toPose(box_pose), + ) + + target_mat = target_pose.to_transformation_matrix() + v1 = (target_mat[:3, :3] @ np.array([[1, 0, 0]]).T).reshape(3) + v2 = box_pose.p - rob_pose.p + v2 = v2 / np.linalg.norm(v2) + axis = np.cross(v1, v2) + angle = np.arccos(np.dot(v1, v2)) + + return rotate_along_axis( + target_pose=target_pose, + center_pose=box_pose, + axis=axis, + theta=angle * theta, + axis_type="world", + towards=-v2, + ) + + +def choose_dirct(block_mat, base_pose: sapien.Pose): + pts = block_mat[:3, :3] @ np.array([[1, -1, 0, 0], [0, 0, 1, -1], [0, 0, 0, 0]]) + dirts = np.sum(np.power(pts - base_pose.p.reshape(3, 1), 2), axis=0) + return pts[:, np.argmin(dirts)] + block_mat[:3, 3] + + +def add_robot_visual_box(task, pose: sapien.Pose | list, name: str = "box"): + box_path = Path("./assets/objects/cube/textured.obj") + if not box_path.exists(): + print("[WARNNING] cube not exists!") + return + + pose = _toPose(pose) + scene: sapien.Scene = task.scene + builder = scene.create_actor_builder() + builder.set_physx_body_type("static") + builder.add_visual_from_file( + filename=str(box_path), + scale=[ + 0.04, + ] * 3, + ) + builder.set_name(name) + builder.set_initial_pose(pose) + return builder.build() + + +def cal_quat_dis(quat1, quat2): + qmult = t3d.quaternions.qmult + qinv = t3d.quaternions.qinverse + qnorm = t3d.quaternions.qnorm + delta_quat = qmult(qinv(quat1), quat2) + return 2 * np.arccos(np.fabs((delta_quat / qnorm(delta_quat))[0])) / np.pi + + +def get_align_matrix(v1: np.ndarray, v2: np.ndarray) -> np.ndarray: + """ + 获取从 v1 到 v2 的旋转矩阵 + """ + v1 = np.array(v1).reshape(3) + v2 = np.array(v2).reshape(3) + + v1 = v1 / np.linalg.norm(v1) + v2 = v2 / np.linalg.norm(v2) + axis = np.cross(v1, v2) + angle = np.arccos(np.dot(v1, v2)) + + if np.linalg.norm(axis) < 1e-6: + return np.eye(3) + else: + return t3d.axangles.axangle2mat(axis, angle) + + +def generate_rotate_vectors( + axis: Literal["x", "y", "z"] | np.ndarray | list, + angle: np.ndarray | list | float, + base: np.ndarray | sapien.Pose | list = None, + vector: np.ndarray | list = [1, 0, 0], +) -> np.ndarray: + """ + 获取从 base 到 axis 的旋转矩阵 + """ + if base is None: + base = np.eye(4) + else: + base = _toPose(base).to_transformation_matrix() + + if isinstance(axis, str): + if axis == "x": + axis = np.array([1, 0, 0]) + elif axis == "y": + axis = np.array([0, 1, 0]) + elif axis == "z": + axis = np.array([0, 0, 1]) + else: + raise ValueError("axis must be x, y or z") + else: + axis = np.array(axis).reshape(3) + + axis = (base[:3, :3] @ axis.reshape(3, 1)).reshape(3) + vector = (base[:3, :3] @ np.array(vector).reshape(3, 1)).reshape(3) + + vector = np.array(vector).reshape((3, 1)) + angle = np.array(angle).flatten() + rotate_mat = np.zeros((3, angle.shape[0])) + for idx, a in enumerate(angle): + rotate_mat[:, idx] = (t3d.axangles.axangle2mat(axis, a) @ vector).reshape(3) + return rotate_mat + + +def get_product_vector(v1: np.ndarray, v2: np.ndarray) -> np.ndarray: + """ + 获取 v2 在 v1 上的投影向量 + """ + v1 = np.array(v1).reshape(3) + v1 = v1 / np.linalg.norm(v1) + v2 = np.array(v2).reshape(3) + return np.dot(v1, v2) * v1 + + +def get_place_pose( + actor_pose: np.ndarray | sapien.Pose | list, + target_pose: np.ndarray | sapien.Pose | list, + constrain: Literal["free", "align"] = "free", + align_axis: list[np.ndarray] | np.ndarray | list = None, + actor_axis: np.ndarray | list = [1, 0, 0], + actor_axis_type: Literal["actor", "world"] = "actor", + z_transform: bool = True, +) -> list: + """ + 获取物体应当被放置到的位置 + 考虑因素: + 1. 三维坐标与给定坐标一致 + 2. 物体的朝向合理 + - 物体 z 轴与给定坐标 z 轴一致 + - 满足在 xy 平面上的一定约束 + - 无约束(直接采用物体当前的 x,y 在 xOy 平面上的投影) + - 物体的 x 轴对齐给定 x 轴 + - 选取物体的 x 轴与给定的世界轴单位向量集合中点积最小的方向 + + actor_pose: 物体当前的 pose + target_pose: 物体应当被放置到的位置 + constrain: 物体的约束类型 + - free: 无约束 + - align: 物体的 x 轴与给定的世界轴向量集合中点积最小的方向 + align_axis: 给定的世界轴向量集合,如果设置为 None,默认使用 target_pose 的 x 轴 + actor_axis: 计算点积的 actor 轴,默认使用 x 轴 + actor_axis_type: actor_axis 的类型,默认使用局部坐标系 + - actor: actor_pose 的局部坐标系 + - world: 世界坐标系 + """ + actor_pose_mat = _toPose(actor_pose).to_transformation_matrix() + target_pose_mat = _toPose(target_pose).to_transformation_matrix() + + # 将物体的三维坐标与给定坐标对齐 + actor_pose_mat[:3, 3] = target_pose_mat[:3, 3] + + target_x = target_pose_mat[:3, 0] + target_y = target_pose_mat[:3, 1] + target_z = target_pose_mat[:3, 2] + + # 将物体的 z 轴与给定坐标的 z 轴对齐 + actor2world = np.array([[1.0, 0.0, 0.0], [0.0, 0.0, -1.0], [0.0, 1.0, 0.0]]).T + if z_transform: + z_align_matrix = get_align_matrix(actor_pose_mat[:3, :3] @ actor2world[:3, 2], target_z) + else: + z_align_matrix = get_align_matrix(actor_pose_mat[:3, 2], target_z) + actor_pose_mat[:3, :3] = z_align_matrix @ actor_pose_mat[:3, :3] + + if constrain == "align": + if align_axis is None: + align_axis = np.array(target_pose_mat[:3, :3] @ np.array([[1, 0, 0]]).T) + elif isinstance(align_axis, list): + align_axis = np.array(align_axis).reshape((-1, 3)).T + else: + align_axis = np.array(align_axis).reshape((3, -1)) + align_axis = align_axis / np.linalg.norm(align_axis, axis=0) + + if actor_axis_type == "actor": + actor_axis = actor_pose_mat[:3, :3] @ np.array(actor_axis).reshape(3, 1) + elif actor_axis_type == "world": + actor_axis = np.array(actor_axis) + closest_axis_id = np.argmax(actor_axis.reshape(3) @ align_axis) + align_axis = align_axis[:, closest_axis_id] + + actor_axis_xOy = get_product_vector(target_x, actor_axis) + get_product_vector(target_y, actor_axis) + align_axis_xOy = get_product_vector(target_x, align_axis) + get_product_vector(target_y, align_axis) + align_mat_xOy = get_align_matrix(actor_axis_xOy, align_axis_xOy) + actor_pose_mat[:3, :3] = align_mat_xOy @ actor_pose_mat[:3, :3] + + return (actor_pose_mat[:3, 3].tolist() + t3d.quaternions.mat2quat(actor_pose_mat[:3, :3]).tolist()) + + +def get_face_prod(q, local_axis, target_axis): + """ + get product of local_axis (under q world) and target_axis + """ + q_mat = t3d.quaternions.quat2mat(q) + face = q_mat @ np.array(local_axis).reshape(3, 1) + face_prod = np.dot(face.reshape(3), np.array(target_axis)) + return face_prod diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/.gitignore b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/.gitignore new file mode 100644 index 00000000..e458af31 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/.gitignore @@ -0,0 +1,146 @@ +bin +logs +wandb +outputs +data +data_local +.vscode +_wandb + +**/.DS_Store + +fuse.cfg + +*.ai + +# Generation results +results/ + +ray/auth.json + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST +act_ckpt/* +!models/* +!detr/models/* + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +act-ckpt/ +processed_data/ \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/LICENSE b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/LICENSE new file mode 100644 index 00000000..35e5f5e2 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Tony Z. Zhao + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/SIM_TASK_CONFIGS.json b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/SIM_TASK_CONFIGS.json new file mode 100644 index 00000000..e69de29b diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/__init__.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/__init__.py new file mode 100644 index 00000000..d4b67709 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/__init__.py @@ -0,0 +1 @@ +from .deploy_policy import * diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/act_policy.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/act_policy.py new file mode 100644 index 00000000..ab627cbf --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/act_policy.py @@ -0,0 +1,219 @@ +import torch.nn as nn +import os +import torch +import numpy as np +import pickle +from torch.nn import functional as F +import torchvision.transforms as transforms + +try: + from detr.main import ( + build_ACT_model_and_optimizer, + build_CNNMLP_model_and_optimizer, + ) +except: + from .detr.main import ( + build_ACT_model_and_optimizer, + build_CNNMLP_model_and_optimizer, + ) +import IPython + +e = IPython.embed + + +class ACTPolicy(nn.Module): + + def __init__(self, args_override, RoboTwin_Config=None): + super().__init__() + model, optimizer = build_ACT_model_and_optimizer(args_override, RoboTwin_Config) + self.model = model # CVAE decoder + self.optimizer = optimizer + self.kl_weight = args_override["kl_weight"] + print(f"KL Weight {self.kl_weight}") + + def __call__(self, qpos, image, actions=None, is_pad=None): + env_state = None + normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + image = normalize(image) + if actions is not None: # training time + actions = actions[:, :self.model.num_queries] + is_pad = is_pad[:, :self.model.num_queries] + + a_hat, is_pad_hat, (mu, logvar) = self.model(qpos, image, env_state, actions, is_pad) + total_kld, dim_wise_kld, mean_kld = kl_divergence(mu, logvar) + loss_dict = dict() + all_l1 = F.l1_loss(actions, a_hat, reduction="none") + l1 = (all_l1 * ~is_pad.unsqueeze(-1)).mean() + loss_dict["l1"] = l1 + loss_dict["kl"] = total_kld[0] + loss_dict["loss"] = loss_dict["l1"] + loss_dict["kl"] * self.kl_weight + return loss_dict + else: # inference time + a_hat, _, (_, _) = self.model(qpos, image, env_state) # no action, sample from prior + return a_hat + + def configure_optimizers(self): + return self.optimizer + + +class CNNMLPPolicy(nn.Module): + + def __init__(self, args_override): + super().__init__() + model, optimizer = build_CNNMLP_model_and_optimizer(args_override) + self.model = model # decoder + self.optimizer = optimizer + + def __call__(self, qpos, image, actions=None, is_pad=None): + env_state = None # TODO + normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + image = normalize(image) + if actions is not None: # training time + actions = actions[:, 0] + a_hat = self.model(qpos, image, env_state, actions) + mse = F.mse_loss(actions, a_hat) + loss_dict = dict() + loss_dict["mse"] = mse + loss_dict["loss"] = loss_dict["mse"] + return loss_dict + else: # inference time + a_hat = self.model(qpos, image, env_state) # no action, sample from prior + return a_hat + + def configure_optimizers(self): + return self.optimizer + + +def kl_divergence(mu, logvar): + batch_size = mu.size(0) + assert batch_size != 0 + if mu.data.ndimension() == 4: + mu = mu.view(mu.size(0), mu.size(1)) + if logvar.data.ndimension() == 4: + logvar = logvar.view(logvar.size(0), logvar.size(1)) + + klds = -0.5 * (1 + logvar - mu.pow(2) - logvar.exp()) + total_kld = klds.sum(1).mean(0, True) + dimension_wise_kld = klds.mean(0) + mean_kld = klds.mean(1).mean(0, True) + + return total_kld, dimension_wise_kld, mean_kld + + +class ACT: + + def __init__(self, args_override=None, RoboTwin_Config=None): + if args_override is None: + args_override = { + "kl_weight": 0.1, # Default value, can be overridden + "device": "cuda:0", + } + self.policy = ACTPolicy(args_override, RoboTwin_Config) + self.device = torch.device(args_override["device"]) + self.policy.to(self.device) + self.policy.eval() + + # Temporal aggregation settings + self.temporal_agg = args_override.get("temporal_agg", False) + self.num_queries = args_override["chunk_size"] + self.state_dim = RoboTwin_Config.action_dim # Standard joint dimension for bimanual robot + self.max_timesteps = 3000 # Large enough for deployment + + # Set query frequency based on temporal_agg - matching imitate_episodes.py logic + self.query_frequency = self.num_queries + if self.temporal_agg: + self.query_frequency = 1 + # Initialize with zeros matching imitate_episodes.py format + self.all_time_actions = torch.zeros([ + self.max_timesteps, + self.max_timesteps + self.num_queries, + self.state_dim, + ]).to(self.device) + print(f"Temporal aggregation enabled with {self.num_queries} queries") + + self.t = 0 # Current timestep + + # Load statistics for normalization + ckpt_dir = args_override.get("ckpt_dir", "") + if ckpt_dir: + # Load dataset stats for normalization + stats_path = os.path.join(ckpt_dir, "dataset_stats.pkl") + if os.path.exists(stats_path): + with open(stats_path, "rb") as f: + self.stats = pickle.load(f) + print(f"Loaded normalization stats from {stats_path}") + else: + print(f"Warning: Could not find stats file at {stats_path}") + self.stats = None + + # Load policy weights + ckpt_path = os.path.join(ckpt_dir, "policy_last.ckpt") + print("current pwd:", os.getcwd()) + if os.path.exists(ckpt_path): + loading_status = self.policy.load_state_dict(torch.load(ckpt_path)) + print(f"Loaded policy weights from {ckpt_path}") + print(f"Loading status: {loading_status}") + else: + print(f"Warning: Could not find policy checkpoint at {ckpt_path}") + else: + self.stats = None + + def pre_process(self, qpos): + """Normalize input joint positions""" + if self.stats is not None: + return (qpos - self.stats["qpos_mean"]) / self.stats["qpos_std"] + return qpos + + def post_process(self, action): + """Denormalize model outputs""" + if self.stats is not None: + return action * self.stats["action_std"] + self.stats["action_mean"] + return action + + def get_action(self, obs=None): + if obs is None: + return None + + # Convert observations to tensors and normalize qpos - matching imitate_episodes.py + qpos_numpy = np.array(obs["qpos"]) + qpos_normalized = self.pre_process(qpos_numpy) + qpos = torch.from_numpy(qpos_normalized).float().to(self.device).unsqueeze(0) + + # Prepare images following imitate_episodes.py pattern + # Stack images from all cameras + curr_images = [] + camera_names = ["head_cam", "left_cam", "right_cam"] + for cam_name in camera_names: + curr_images.append(obs[cam_name]) + curr_image = np.stack(curr_images, axis=0) + curr_image = torch.from_numpy(curr_image).float().to(self.device).unsqueeze(0) + + with torch.no_grad(): + # Only query the policy at specified intervals - exactly like imitate_episodes.py + if self.t % self.query_frequency == 0: + self.all_actions = self.policy(qpos, curr_image) + + if self.temporal_agg: + # Match temporal aggregation exactly from imitate_episodes.py + self.all_time_actions[[self.t], self.t:self.t + self.num_queries] = (self.all_actions) + actions_for_curr_step = self.all_time_actions[:, self.t] + actions_populated = torch.all(actions_for_curr_step != 0, axis=1) + actions_for_curr_step = actions_for_curr_step[actions_populated] + + # Use same weighting factor as in imitate_episodes.py + k = 0.01 + exp_weights = np.exp(-k * np.arange(len(actions_for_curr_step))) + exp_weights = exp_weights / exp_weights.sum() + exp_weights = (torch.from_numpy(exp_weights).to(self.device).unsqueeze(dim=1)) + + raw_action = (actions_for_curr_step * exp_weights).sum(dim=0, keepdim=True) + else: + # Direct action selection, same as imitate_episodes.py + raw_action = self.all_actions[:, self.t % self.query_frequency] + + # Denormalize action + raw_action = raw_action.cpu().numpy() + action = self.post_process(raw_action) + + self.t += 1 + return action \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/conda_env.yaml b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/conda_env.yaml new file mode 100644 index 00000000..0f44d6b0 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/conda_env.yaml @@ -0,0 +1,23 @@ +name: aloha +channels: + - pytorch + - nvidia + - conda-forge +dependencies: + - python=3.9 + - pip=23.0.1 + - pytorch=2.0.0 + - torchvision=0.15.0 + - pytorch-cuda=11.8 + - pyquaternion=0.9.9 + - pyyaml=6.0 + - rospkg=1.5.0 + - pexpect=4.8.0 + - mujoco=2.3.3 + - dm_control=1.0.9 + - py-opencv=4.7.0 + - matplotlib=3.7.1 + - einops=0.6.0 + - packaging=23.0 + - h5py=3.8.0 + - ipython=8.12.0 diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/constants.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/constants.py new file mode 100644 index 00000000..97301cf8 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/constants.py @@ -0,0 +1,88 @@ +import pathlib +import os, json + +current_dir = os.path.dirname(__file__) + +### Task parameters +SIM_TASK_CONFIGS_PATH = os.path.join(current_dir, "./SIM_TASK_CONFIGS.json") +with open(SIM_TASK_CONFIGS_PATH, "r") as f: + SIM_TASK_CONFIGS = json.load(f) + +### Simulation envs fixed constants +DT = 0.02 +JOINT_NAMES = [ + "waist", + "shoulder", + "elbow", + "forearm_roll", + "wrist_angle", + "wrist_rotate", +] +START_ARM_POSE = [ + 0, + -0.96, + 1.16, + 0, + -0.3, + 0, + 0.02239, + -0.02239, + 0, + -0.96, + 1.16, + 0, + -0.3, + 0, + 0.02239, + -0.02239, +] + +XML_DIR = (str(pathlib.Path(__file__).parent.resolve()) + "/assets/") # note: absolute path + +# Left finger position limits (qpos[7]), right_finger = -1 * left_finger +MASTER_GRIPPER_POSITION_OPEN = 0.02417 +MASTER_GRIPPER_POSITION_CLOSE = 0.01244 +PUPPET_GRIPPER_POSITION_OPEN = 0.05800 +PUPPET_GRIPPER_POSITION_CLOSE = 0.01844 + +# Gripper joint limits (qpos[6]) +MASTER_GRIPPER_JOINT_OPEN = 0.3083 +MASTER_GRIPPER_JOINT_CLOSE = -0.6842 +PUPPET_GRIPPER_JOINT_OPEN = 1.4910 +PUPPET_GRIPPER_JOINT_CLOSE = -0.6213 + +############################ Helper functions ############################ + +MASTER_GRIPPER_POSITION_NORMALIZE_FN = lambda x: (x - MASTER_GRIPPER_POSITION_CLOSE) / (MASTER_GRIPPER_POSITION_OPEN - + MASTER_GRIPPER_POSITION_CLOSE) +PUPPET_GRIPPER_POSITION_NORMALIZE_FN = lambda x: (x - PUPPET_GRIPPER_POSITION_CLOSE) / (PUPPET_GRIPPER_POSITION_OPEN - + PUPPET_GRIPPER_POSITION_CLOSE) +MASTER_GRIPPER_POSITION_UNNORMALIZE_FN = ( + lambda x: x * (MASTER_GRIPPER_POSITION_OPEN - MASTER_GRIPPER_POSITION_CLOSE) + MASTER_GRIPPER_POSITION_CLOSE) +PUPPET_GRIPPER_POSITION_UNNORMALIZE_FN = ( + lambda x: x * (PUPPET_GRIPPER_POSITION_OPEN - PUPPET_GRIPPER_POSITION_CLOSE) + PUPPET_GRIPPER_POSITION_CLOSE) +MASTER2PUPPET_POSITION_FN = lambda x: PUPPET_GRIPPER_POSITION_UNNORMALIZE_FN(MASTER_GRIPPER_POSITION_NORMALIZE_FN(x)) + +MASTER_GRIPPER_JOINT_NORMALIZE_FN = lambda x: (x - MASTER_GRIPPER_JOINT_CLOSE) / (MASTER_GRIPPER_JOINT_OPEN - + MASTER_GRIPPER_JOINT_CLOSE) +PUPPET_GRIPPER_JOINT_NORMALIZE_FN = lambda x: (x - PUPPET_GRIPPER_JOINT_CLOSE) / (PUPPET_GRIPPER_JOINT_OPEN - + PUPPET_GRIPPER_JOINT_CLOSE) +MASTER_GRIPPER_JOINT_UNNORMALIZE_FN = ( + lambda x: x * (MASTER_GRIPPER_JOINT_OPEN - MASTER_GRIPPER_JOINT_CLOSE) + MASTER_GRIPPER_JOINT_CLOSE) +PUPPET_GRIPPER_JOINT_UNNORMALIZE_FN = ( + lambda x: x * (PUPPET_GRIPPER_JOINT_OPEN - PUPPET_GRIPPER_JOINT_CLOSE) + PUPPET_GRIPPER_JOINT_CLOSE) +MASTER2PUPPET_JOINT_FN = lambda x: PUPPET_GRIPPER_JOINT_UNNORMALIZE_FN(MASTER_GRIPPER_JOINT_NORMALIZE_FN(x)) + +MASTER_GRIPPER_VELOCITY_NORMALIZE_FN = lambda x: x / (MASTER_GRIPPER_POSITION_OPEN - MASTER_GRIPPER_POSITION_CLOSE) +PUPPET_GRIPPER_VELOCITY_NORMALIZE_FN = lambda x: x / (PUPPET_GRIPPER_POSITION_OPEN - PUPPET_GRIPPER_POSITION_CLOSE) + +MASTER_POS2JOINT = (lambda x: MASTER_GRIPPER_POSITION_NORMALIZE_FN(x) * + (MASTER_GRIPPER_JOINT_OPEN - MASTER_GRIPPER_JOINT_CLOSE) + MASTER_GRIPPER_JOINT_CLOSE) +MASTER_JOINT2POS = lambda x: MASTER_GRIPPER_POSITION_UNNORMALIZE_FN( + (x - MASTER_GRIPPER_JOINT_CLOSE) / (MASTER_GRIPPER_JOINT_OPEN - MASTER_GRIPPER_JOINT_CLOSE)) +PUPPET_POS2JOINT = (lambda x: PUPPET_GRIPPER_POSITION_NORMALIZE_FN(x) * + (PUPPET_GRIPPER_JOINT_OPEN - PUPPET_GRIPPER_JOINT_CLOSE) + PUPPET_GRIPPER_JOINT_CLOSE) +PUPPET_JOINT2POS = lambda x: PUPPET_GRIPPER_POSITION_UNNORMALIZE_FN( + (x - PUPPET_GRIPPER_JOINT_CLOSE) / (PUPPET_GRIPPER_JOINT_OPEN - PUPPET_GRIPPER_JOINT_CLOSE)) + +MASTER_GRIPPER_JOINT_MID = (MASTER_GRIPPER_JOINT_OPEN + MASTER_GRIPPER_JOINT_CLOSE) / 2 diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/deploy_policy.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/deploy_policy.py new file mode 100644 index 00000000..88fef7c9 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/deploy_policy.py @@ -0,0 +1,57 @@ +import sys +import numpy as np +import torch +import os +import pickle +import cv2 +import time # Add import for timestamp +import h5py # Add import for HDF5 +from datetime import datetime # Add import for datetime formatting +from .act_policy import ACT +import copy +from argparse import Namespace + +def encode_obs(observation): + head_cam = cv2.resize(observation["observation"]["head_camera"]["rgb"], (640, 480), interpolation=cv2.INTER_LINEAR) + left_cam = cv2.resize(observation["observation"]["left_camera"]["rgb"], (640, 480), interpolation=cv2.INTER_LINEAR) + right_cam = cv2.resize(observation["observation"]["right_camera"]["rgb"], (640, 480), interpolation=cv2.INTER_LINEAR) + head_cam = np.moveaxis(head_cam, -1, 0) / 255.0 + left_cam = np.moveaxis(left_cam, -1, 0) / 255.0 + right_cam = np.moveaxis(right_cam, -1, 0) / 255.0 + qpos = (observation["joint_action"]["left_arm"] + [observation["joint_action"]["left_gripper"]] + + observation["joint_action"]["right_arm"] + [observation["joint_action"]["right_gripper"]]) + return { + "head_cam": head_cam, + "left_cam": left_cam, + "right_cam": right_cam, + "qpos": qpos, + } + +def get_model(usr_args): + return ACT(usr_args, Namespace(**usr_args)) + + +def eval(TASK_ENV, model, observation): + obs = encode_obs(observation) + # instruction = TASK_ENV.get_instruction() + + # Get action from model + actions = model.get_action(obs) + for action in actions: + TASK_ENV.take_action(action) + observation = TASK_ENV.get_obs() + return observation + + +def reset_model(model): + # Reset temporal aggregation state if enabled + if model.temporal_agg: + model.all_time_actions = torch.zeros([ + model.max_timesteps, + model.max_timesteps + model.num_queries, + model.state_dim, + ]).to(model.device) + model.t = 0 + print("Reset temporal aggregation state") + else: + model.t = 0 diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/deploy_policy.yml b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/deploy_policy.yml new file mode 100644 index 00000000..245f158e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/deploy_policy.yml @@ -0,0 +1,39 @@ +# Basic experiment configuration +task_name: null +policy_name: ACT +task_config: null +ckpt_setting: null +seed: 0 +instruction_type: unseen + +# ACT-specific arguments +action_dim: 14 +kl_weight: 10.0 +chunk_size: 50 +hidden_dim: 512 +dim_feedforward: 3200 +temporal_agg: false +device: cuda:0 + +# DETR parser args +ckpt_dir: null +policy_class: ACT +num_epochs: 2000 + +# Model training params +position_embedding: sine +lr_backbone: 0.00001 +weight_decay: 0.0001 +lr: 0.00001 +masks: false +dilation: false +backbone: resnet18 +nheads: 8 +enc_layers: 4 +dec_layers: 7 +pre_norm: false +dropout: 0.1 +camera_names: + - cam_high + - cam_right_wrist + - cam_left_wrist diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/.gitignore b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/.gitignore new file mode 100644 index 00000000..01a4eef4 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/.gitignore @@ -0,0 +1 @@ +!models \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/LICENSE b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/LICENSE new file mode 100644 index 00000000..b1395e94 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2020 - present, Facebook, Inc + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/README.md b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/README.md new file mode 100644 index 00000000..500b1b8d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/README.md @@ -0,0 +1,9 @@ +This part of the codebase is modified from DETR https://github.com/facebookresearch/detr under APACHE 2.0. + + @article{Carion2020EndtoEndOD, + title={End-to-End Object Detection with Transformers}, + author={Nicolas Carion and Francisco Massa and Gabriel Synnaeve and Nicolas Usunier and Alexander Kirillov and Sergey Zagoruyko}, + journal={ArXiv}, + year={2020}, + volume={abs/2005.12872} + } \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/main.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/main.py new file mode 100644 index 00000000..1fda8a23 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/main.py @@ -0,0 +1,174 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import argparse +from pathlib import Path + +import numpy as np +import torch +from .models import build_ACT_model, build_CNNMLP_model + +import IPython + +e = IPython.embed + + +def get_args_parser(): + parser = argparse.ArgumentParser("Set transformer detector", add_help=False) + parser.add_argument("--lr", default=1e-4, type=float) # will be overridden + parser.add_argument("--lr_backbone", default=1e-5, type=float) # will be overridden + parser.add_argument("--batch_size", default=2, type=int) # not used + parser.add_argument("--weight_decay", default=1e-4, type=float) + parser.add_argument("--epochs", default=300, type=int) # not used + parser.add_argument("--lr_drop", default=200, type=int) # not used + parser.add_argument( + "--clip_max_norm", + default=0.1, + type=float, # not used + help="gradient clipping max norm", + ) + + # Model parameters + # * Backbone + parser.add_argument( + "--backbone", + default="resnet18", + type=str, # will be overridden + help="Name of the convolutional backbone to use", + ) + parser.add_argument( + "--dilation", + action="store_true", + help="If true, we replace stride with dilation in the last convolutional block (DC5)", + ) + parser.add_argument( + "--position_embedding", + default="sine", + type=str, + choices=("sine", "learned"), + help="Type of positional embedding to use on top of the image features", + ) + parser.add_argument( + "--camera_names", + default=[], + type=list, # will be overridden + help="A list of camera names", + ) + + # * Transformer + parser.add_argument( + "--enc_layers", + default=4, + type=int, # will be overridden + help="Number of encoding layers in the transformer", + ) + parser.add_argument( + "--dec_layers", + default=6, + type=int, # will be overridden + help="Number of decoding layers in the transformer", + ) + parser.add_argument( + "--dim_feedforward", + default=2048, + type=int, # will be overridden + help="Intermediate size of the feedforward layers in the transformer blocks", + ) + parser.add_argument( + "--hidden_dim", + default=256, + type=int, # will be overridden + help="Size of the embeddings (dimension of the transformer)", + ) + parser.add_argument("--dropout", default=0.1, type=float, help="Dropout applied in the transformer") + parser.add_argument( + "--nheads", + default=8, + type=int, # will be overridden + help="Number of attention heads inside the transformer's attentions", + ) + # parser.add_argument('--num_queries', required=True, type=int, # will be overridden + # help="Number of query slots")#AGGSIZE + parser.add_argument("--pre_norm", action="store_true") + + # * Segmentation + parser.add_argument( + "--masks", + action="store_true", + help="Train segmentation head if the flag is provided", + ) + + # repeat args in imitate_episodes just to avoid error. Will not be used + parser.add_argument("--eval", action="store_true") + parser.add_argument("--onscreen_render", action="store_true") + parser.add_argument("--ckpt_dir", action="store", type=str, help="ckpt_dir", required=True) + parser.add_argument( + "--policy_class", + action="store", + type=str, + help="policy_class, capitalize", + required=True, + ) + parser.add_argument("--task_name", action="store", type=str, help="task_name", required=True) + parser.add_argument("--seed", action="store", type=int, help="seed", required=True) + parser.add_argument("--num_epochs", action="store", type=int, help="num_epochs", required=True) + parser.add_argument("--kl_weight", action="store", type=int, help="KL Weight", required=False) + parser.add_argument("--chunk_size", action="store", type=int, help="chunk_size", required=False) + parser.add_argument("--temporal_agg", action="store_true") + parser.add_argument("--state_dim", action="store", type=int, help="state dim", required=True) + parser.add_argument("--save_freq", action="store", type=int, help="save ckpt frequency", required=False, default=6000) + # parser.add_argument('--num_queries',type=int, required=True) + # parser.add_argument('--actionsByQuery',type=int, required=True) + + return parser + + +def build_ACT_model_and_optimizer(args_override, RoboTwin_Config=None): + if RoboTwin_Config is None: + parser = argparse.ArgumentParser("DETR training and evaluation script", parents=[get_args_parser()]) + args = parser.parse_args() + for k, v in args_override.items(): + setattr(args, k, v) + else: + args = RoboTwin_Config + + print("build_ACT_model_and_optimizer", args) + + print(args) + model = build_ACT_model(args) + model.cuda() + + param_dicts = [ + { + "params": [p for n, p in model.named_parameters() if "backbone" not in n and p.requires_grad] + }, + { + "params": [p for n, p in model.named_parameters() if "backbone" in n and p.requires_grad], + "lr": args.lr_backbone, + }, + ] + optimizer = torch.optim.AdamW(param_dicts, lr=args.lr, weight_decay=args.weight_decay) + + return model, optimizer + + +def build_CNNMLP_model_and_optimizer(args_override): + parser = argparse.ArgumentParser("DETR training and evaluation script", parents=[get_args_parser()]) + args = parser.parse_args() + + for k, v in args_override.items(): + setattr(args, k, v) + + model = build_CNNMLP_model(args) + model.cuda() + + param_dicts = [ + { + "params": [p for n, p in model.named_parameters() if "backbone" not in n and p.requires_grad] + }, + { + "params": [p for n, p in model.named_parameters() if "backbone" in n and p.requires_grad], + "lr": args.lr_backbone, + }, + ] + optimizer = torch.optim.AdamW(param_dicts, lr=args.lr, weight_decay=args.weight_decay) + + return model, optimizer diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/models/__init__.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/models/__init__.py new file mode 100644 index 00000000..0f69693e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/models/__init__.py @@ -0,0 +1,11 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +from .detr_vae import build as build_vae +from .detr_vae import build_cnnmlp as build_cnnmlp + + +def build_ACT_model(args): + return build_vae(args) + + +def build_CNNMLP_model(args): + return build_cnnmlp(args) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/models/backbone.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/models/backbone.py new file mode 100644 index 00000000..85b324c2 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/models/backbone.py @@ -0,0 +1,128 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Backbone modules. +""" +from collections import OrderedDict +import os +import torch +import torch.nn.functional as F +import torchvision +from torch import nn +from torchvision.models._utils import IntermediateLayerGetter +from typing import Dict, List +import sys + +current_dir = os.path.dirname(os.path.abspath(__file__)) +project_root = os.path.abspath(os.path.join(current_dir, '..')) +sys.path.append(project_root) + +from util.misc import NestedTensor, is_main_process + +from .position_encoding import build_position_encoding + +import IPython + +e = IPython.embed + + +class FrozenBatchNorm2d(torch.nn.Module): + """ + BatchNorm2d where the batch statistics and the affine parameters are fixed. + + Copy-paste from torchvision.misc.ops with added eps before rqsrt, + without which any other policy_models than torchvision.policy_models.resnet[18,34,50,101] + produce nans. + """ + + def __init__(self, n): + super(FrozenBatchNorm2d, self).__init__() + self.register_buffer("weight", torch.ones(n)) + self.register_buffer("bias", torch.zeros(n)) + self.register_buffer("running_mean", torch.zeros(n)) + self.register_buffer("running_var", torch.ones(n)) + + def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, + error_msgs): + num_batches_tracked_key = prefix + 'num_batches_tracked' + if num_batches_tracked_key in state_dict: + del state_dict[num_batches_tracked_key] + + super(FrozenBatchNorm2d, self)._load_from_state_dict(state_dict, prefix, local_metadata, strict, missing_keys, + unexpected_keys, error_msgs) + + def forward(self, x): + # move reshapes to the beginning + # to make it fuser-friendly + w = self.weight.reshape(1, -1, 1, 1) + b = self.bias.reshape(1, -1, 1, 1) + rv = self.running_var.reshape(1, -1, 1, 1) + rm = self.running_mean.reshape(1, -1, 1, 1) + eps = 1e-5 + scale = w * (rv + eps).rsqrt() + bias = b - rm * scale + return x * scale + bias + + +class BackboneBase(nn.Module): + + def __init__(self, backbone: nn.Module, train_backbone: bool, num_channels: int, return_interm_layers: bool): + super().__init__() + # for name, parameter in backbone.named_parameters(): # only train later layers # TODO do we want this? + # if not train_backbone or 'layer2' not in name and 'layer3' not in name and 'layer4' not in name: + # parameter.requires_grad_(False) + if return_interm_layers: + return_layers = {"layer1": "0", "layer2": "1", "layer3": "2", "layer4": "3"} + else: + return_layers = {'layer4': "0"} + self.body = IntermediateLayerGetter(backbone, return_layers=return_layers) + self.num_channels = num_channels + + def forward(self, tensor): + xs = self.body(tensor) + return xs + # out: Dict[str, NestedTensor] = {} + # for name, x in xs.items(): + # m = tensor_list.mask + # assert m is not None + # mask = F.interpolate(m[None].float(), size=x.shape[-2:]).to(torch.bool)[0] + # out[name] = NestedTensor(x, mask) + # return out + + +class Backbone(BackboneBase): + """ResNet backbone with frozen BatchNorm.""" + + def __init__(self, name: str, train_backbone: bool, return_interm_layers: bool, dilation: bool): + backbone = getattr(torchvision.models, + name)(replace_stride_with_dilation=[False, False, dilation], + pretrained=is_main_process(), + norm_layer=FrozenBatchNorm2d) # pretrained # TODO do we want frozen batch_norm?? + num_channels = 512 if name in ('resnet18', 'resnet34') else 2048 + super().__init__(backbone, train_backbone, num_channels, return_interm_layers) + + +class Joiner(nn.Sequential): + + def __init__(self, backbone, position_embedding): + super().__init__(backbone, position_embedding) + + def forward(self, tensor_list: NestedTensor): + xs = self[0](tensor_list) + out: List[NestedTensor] = [] + pos = [] + for name, x in xs.items(): + out.append(x) + # position encoding + pos.append(self[1](x).to(x.dtype)) + + return out, pos + + +def build_backbone(args): + position_embedding = build_position_encoding(args) + train_backbone = args.lr_backbone > 0 + return_interm_layers = args.masks + backbone = Backbone(args.backbone, train_backbone, return_interm_layers, args.dilation) + model = Joiner(backbone, position_embedding) + model.num_channels = backbone.num_channels + return model diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/models/detr_vae.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/models/detr_vae.py new file mode 100644 index 00000000..a45f36d3 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/models/detr_vae.py @@ -0,0 +1,281 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +DETR model and criterion classes. +""" +import torch +from torch import nn +from torch.autograd import Variable +from .backbone import build_backbone +from .transformer import build_transformer, TransformerEncoder, TransformerEncoderLayer + +import numpy as np + +import IPython + +e = IPython.embed + + +def reparametrize(mu, logvar): + std = logvar.div(2).exp() + eps = Variable(std.data.new(std.size()).normal_()) + return mu + std * eps + + +def get_sinusoid_encoding_table(n_position, d_hid): + + def get_position_angle_vec(position): + return [position / np.power(10000, 2 * (hid_j // 2) / d_hid) for hid_j in range(d_hid)] + + sinusoid_table = np.array([get_position_angle_vec(pos_i) for pos_i in range(n_position)]) + sinusoid_table[:, 0::2] = np.sin(sinusoid_table[:, 0::2]) # dim 2i + sinusoid_table[:, 1::2] = np.cos(sinusoid_table[:, 1::2]) # dim 2i+1 + + return torch.FloatTensor(sinusoid_table).unsqueeze(0) + + +class DETRVAE(nn.Module): + """ This is the DETR module that performs object detection """ + + def __init__(self, backbones, transformer, encoder, state_dim, num_queries, camera_names): + """ Initializes the model. + Parameters: + backbones: torch module of the backbone to be used. See backbone.py + transformer: torch module of the transformer architecture. See transformer.py + state_dim: robot state dimension of the environment + num_queries: number of object queries, ie detection slot. This is the maximal number of objects + DETR can detect in a single image. For COCO, we recommend 100 queries. + aux_loss: True if auxiliary decoding losses (loss at each decoder layer) are to be used. + """ + super().__init__() + self.num_queries = num_queries + self.camera_names = camera_names + self.transformer = transformer + self.encoder = encoder + hidden_dim = transformer.d_model + self.action_head = nn.Linear(hidden_dim, state_dim) + self.is_pad_head = nn.Linear(hidden_dim, 1) + self.query_embed = nn.Embedding(num_queries, hidden_dim) + if backbones is not None: + self.input_proj = nn.Conv2d(backbones[0].num_channels, hidden_dim, kernel_size=1) + self.backbones = nn.ModuleList(backbones) + self.input_proj_robot_state = nn.Linear(state_dim, hidden_dim) + else: + # input_dim = 14 + 7 # robot_state + env_state + self.input_proj_robot_state = nn.Linear(state_dim, hidden_dim) + self.input_proj_env_state = nn.Linear(7, hidden_dim) + self.pos = torch.nn.Embedding(2, hidden_dim) + self.backbones = None + + # encoder extra parameters + self.latent_dim = 32 # final size of latent z # TODO tune + self.cls_embed = nn.Embedding(1, hidden_dim) # extra cls token embedding + self.encoder_action_proj = nn.Linear(state_dim, hidden_dim) # project action to embedding + self.encoder_joint_proj = nn.Linear(state_dim, hidden_dim) # project qpos to embedding + self.latent_proj = nn.Linear(hidden_dim, self.latent_dim * 2) # project hidden state to latent std, var + self.register_buffer('pos_table', get_sinusoid_encoding_table(1 + 1 + num_queries, + hidden_dim)) # [CLS], qpos, a_seq + + # decoder extra parameters + self.latent_out_proj = nn.Linear(self.latent_dim, hidden_dim) # project latent sample to embedding + self.additional_pos_embed = nn.Embedding(2, hidden_dim) # learned position embedding for proprio and latent + + def forward(self, qpos, image, env_state, actions=None, is_pad=None): + """ + qpos: batch, qpos_dim + image: batch, num_cam, channel, height, width + env_state: None + actions: batch, seq, action_dim + """ + is_training = actions is not None # train or val + bs, _ = qpos.shape + ### Obtain latent z from action sequence + if is_training: + # project action sequence to embedding dim, and concat with a CLS token + action_embed = self.encoder_action_proj(actions) # (bs, seq, hidden_dim) + qpos_embed = self.encoder_joint_proj(qpos) # (bs, hidden_dim) + qpos_embed = torch.unsqueeze(qpos_embed, axis=1) # (bs, 1, hidden_dim) + cls_embed = self.cls_embed.weight # (1, hidden_dim) + cls_embed = torch.unsqueeze(cls_embed, axis=0).repeat(bs, 1, 1) # (bs, 1, hidden_dim) + encoder_input = torch.cat([cls_embed, qpos_embed, action_embed], axis=1) # (bs, seq+1, hidden_dim) + encoder_input = encoder_input.permute(1, 0, 2) # (seq+1, bs, hidden_dim) + # do not mask cls token + cls_joint_is_pad = torch.full((bs, 2), False).to(qpos.device) # False: not a padding + is_pad = torch.cat([cls_joint_is_pad, is_pad], axis=1) # (bs, seq+1) + # obtain position embedding + pos_embed = self.pos_table.clone().detach() + pos_embed = pos_embed.permute(1, 0, 2) # (seq+1, 1, hidden_dim) + # query model + encoder_output = self.encoder(encoder_input, pos=pos_embed, src_key_padding_mask=is_pad) + encoder_output = encoder_output[0] # take cls output only + latent_info = self.latent_proj(encoder_output) + mu = latent_info[:, :self.latent_dim] + logvar = latent_info[:, self.latent_dim:] + latent_sample = reparametrize(mu, logvar) + latent_input = self.latent_out_proj(latent_sample) + else: + mu = logvar = None + latent_sample = torch.zeros([bs, self.latent_dim], dtype=torch.float32).to(qpos.device) + latent_input = self.latent_out_proj(latent_sample) + + if self.backbones is not None: + # Image observation features and position embeddings + all_cam_features = [] + all_cam_pos = [] + # print("image.shape in detr_vae", image.shape,"camera_names", self.camera_names) + for cam_id, cam_name in enumerate(self.camera_names): + # print("cam_id", cam_id, "cam_name", cam_name) + features, pos = self.backbones[0](image[:, cam_id]) # HARDCODED + features = features[0] # take the last layer feature + pos = pos[0] + all_cam_features.append(self.input_proj(features)) + all_cam_pos.append(pos) + # proprioception features + proprio_input = self.input_proj_robot_state(qpos) + # fold camera dimension into width dimension + src = torch.cat(all_cam_features, axis=3) + pos = torch.cat(all_cam_pos, axis=3) + hs = self.transformer(src, None, self.query_embed.weight, pos, latent_input, proprio_input, + self.additional_pos_embed.weight)[0] + else: + qpos = self.input_proj_robot_state(qpos) + env_state = self.input_proj_env_state(env_state) + transformer_input = torch.cat([qpos, env_state], axis=1) # seq length = 2 + hs = self.transformer(transformer_input, None, self.query_embed.weight, self.pos.weight)[0] + a_hat = self.action_head(hs) + is_pad_hat = self.is_pad_head(hs) + return a_hat, is_pad_hat, [mu, logvar] + + +class CNNMLP(nn.Module): + + def __init__(self, backbones, state_dim, camera_names): + """ Initializes the model. + Parameters: + backbones: torch module of the backbone to be used. See backbone.py + transformer: torch module of the transformer architecture. See transformer.py + state_dim: robot state dimension of the environment + num_queries: number of object queries, ie detection slot. This is the maximal number of objects + DETR can detect in a single image. For COCO, we recommend 100 queries. + aux_loss: True if auxiliary decoding losses (loss at each decoder layer) are to be used. + """ + super().__init__() + self.camera_names = camera_names + self.action_head = nn.Linear(1000, state_dim) # TODO add more + if backbones is not None: + self.backbones = nn.ModuleList(backbones) + backbone_down_projs = [] + for backbone in backbones: + down_proj = nn.Sequential(nn.Conv2d(backbone.num_channels, 128, kernel_size=5), + nn.Conv2d(128, 64, kernel_size=5), nn.Conv2d(64, 32, kernel_size=5)) + backbone_down_projs.append(down_proj) + self.backbone_down_projs = nn.ModuleList(backbone_down_projs) + + mlp_in_dim = 768 * len(backbones) + 14 + self.mlp = mlp(input_dim=mlp_in_dim, hidden_dim=1024, output_dim=state_dim, hidden_depth=2) + else: + raise NotImplementedError + + def forward(self, qpos, image, env_state, actions=None): + """ + qpos: batch, qpos_dim + image: batch, num_cam, channel, height, width + env_state: None + actions: batch, seq, action_dim + """ + is_training = actions is not None # train or val + bs, _ = qpos.shape + # Image observation features and position embeddings + all_cam_features = [] + for cam_id, cam_name in enumerate(self.camera_names): + features, pos = self.backbones[cam_id](image[:, cam_id]) + features = features[0] # take the last layer feature + pos = pos[0] # not used + all_cam_features.append(self.backbone_down_projs[cam_id](features)) + # flatten everything + flattened_features = [] + for cam_feature in all_cam_features: + flattened_features.append(cam_feature.reshape([bs, -1])) + flattened_features = torch.cat(flattened_features, axis=1) # 768 each + features = torch.cat([flattened_features, qpos], axis=1) # qpos: 14 + a_hat = self.mlp(features) + return a_hat + + +def mlp(input_dim, hidden_dim, output_dim, hidden_depth): + if hidden_depth == 0: + mods = [nn.Linear(input_dim, output_dim)] + else: + mods = [nn.Linear(input_dim, hidden_dim), nn.ReLU(inplace=True)] + for i in range(hidden_depth - 1): + mods += [nn.Linear(hidden_dim, hidden_dim), nn.ReLU(inplace=True)] + mods.append(nn.Linear(hidden_dim, output_dim)) + trunk = nn.Sequential(*mods) + return trunk + + +def build_encoder(args): + d_model = args.hidden_dim # 256 + dropout = args.dropout # 0.1 + nhead = args.nheads # 8 + dim_feedforward = args.dim_feedforward # 2048 + num_encoder_layers = args.enc_layers # 4 # TODO shared with VAE decoder + normalize_before = args.pre_norm # False + activation = "relu" + + encoder_layer = TransformerEncoderLayer(d_model, nhead, dim_feedforward, dropout, activation, normalize_before) + encoder_norm = nn.LayerNorm(d_model) if normalize_before else None + encoder = TransformerEncoder(encoder_layer, num_encoder_layers, encoder_norm) + + return encoder + + +def build(args): + state_dim = 14 # TODO hardcode + + # From state + # backbone = None # from state for now, no need for conv nets + # From image + backbones = [] + backbone = build_backbone(args) + backbones.append(backbone) + + transformer = build_transformer(args) + + encoder = build_encoder(args) + + model = DETRVAE( + backbones, + transformer, + encoder, + state_dim=state_dim, + num_queries=args.chunk_size, #gyh + camera_names=args.camera_names, + ) + + n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad) + print("number of parameters: %.2fM" % (n_parameters / 1e6, )) + + return model + + +def build_cnnmlp(args): + state_dim = 16 # TODO hardcode + + # From state + # backbone = None # from state for now, no need for conv nets + # From image + backbones = [] + for _ in args.camera_names: + backbone = build_backbone(args) + backbones.append(backbone) + + model = CNNMLP( + backbones, + state_dim=state_dim, + camera_names=args.camera_names, + ) + + n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad) + print("number of parameters: %.2fM" % (n_parameters / 1e6, )) + + return model diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/models/position_encoding.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/models/position_encoding.py new file mode 100644 index 00000000..87e0e978 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/models/position_encoding.py @@ -0,0 +1,98 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Various positional encodings for the transformer. +""" +import math +import torch +from torch import nn + +from util.misc import NestedTensor + +import IPython + +e = IPython.embed + + +class PositionEmbeddingSine(nn.Module): + """ + This is a more standard version of the position embedding, very similar to the one + used by the Attention is all you need paper, generalized to work on images. + """ + + def __init__(self, num_pos_feats=64, temperature=10000, normalize=False, scale=None): + super().__init__() + self.num_pos_feats = num_pos_feats + self.temperature = temperature + self.normalize = normalize + if scale is not None and normalize is False: + raise ValueError("normalize should be True if scale is passed") + if scale is None: + scale = 2 * math.pi + self.scale = scale + + def forward(self, tensor): + x = tensor + # mask = tensor_list.mask + # assert mask is not None + # not_mask = ~mask + + not_mask = torch.ones_like(x[0, [0]]) + y_embed = not_mask.cumsum(1, dtype=torch.float32) + x_embed = not_mask.cumsum(2, dtype=torch.float32) + if self.normalize: + eps = 1e-6 + y_embed = y_embed / (y_embed[:, -1:, :] + eps) * self.scale + x_embed = x_embed / (x_embed[:, :, -1:] + eps) * self.scale + + dim_t = torch.arange(self.num_pos_feats, dtype=torch.float32, device=x.device) + dim_t = self.temperature**(2 * (dim_t // 2) / self.num_pos_feats) + + pos_x = x_embed[:, :, :, None] / dim_t + pos_y = y_embed[:, :, :, None] / dim_t + pos_x = torch.stack((pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4).flatten(3) + pos_y = torch.stack((pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4).flatten(3) + pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2) + return pos + + +class PositionEmbeddingLearned(nn.Module): + """ + Absolute pos embedding, learned. + """ + + def __init__(self, num_pos_feats=256): + super().__init__() + self.row_embed = nn.Embedding(50, num_pos_feats) + self.col_embed = nn.Embedding(50, num_pos_feats) + self.reset_parameters() + + def reset_parameters(self): + nn.init.uniform_(self.row_embed.weight) + nn.init.uniform_(self.col_embed.weight) + + def forward(self, tensor_list: NestedTensor): + x = tensor_list.tensors + h, w = x.shape[-2:] + i = torch.arange(w, device=x.device) + j = torch.arange(h, device=x.device) + x_emb = self.col_embed(i) + y_emb = self.row_embed(j) + pos = torch.cat([ + x_emb.unsqueeze(0).repeat(h, 1, 1), + y_emb.unsqueeze(1).repeat(1, w, 1), + ], dim=-1).permute(2, 0, 1).unsqueeze(0).repeat(x.shape[0], 1, 1, 1) + return pos + + +def build_position_encoding(args): + # print(args.keys()) + N_steps = args.hidden_dim // 2 + if args.position_embedding in ('v2', 'sine'): + # TODO find a better way of exposing other arguments + position_embedding = PositionEmbeddingSine(N_steps, normalize=True) + elif args.position_embedding in ('v3', 'learned'): + position_embedding = PositionEmbeddingLearned(N_steps) + else: + raise ValueError(f"not supported {args.position_embedding}") + + return position_embedding diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/models/transformer.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/models/transformer.py new file mode 100644 index 00000000..5e2e400f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/models/transformer.py @@ -0,0 +1,338 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +DETR Transformer class. + +Copy-paste from torch.nn.Transformer with modifications: + * positional encodings are passed in MHattention + * extra LN at the end of encoder is removed + * decoder returns a stack of activations from all decoding layers +""" +import copy +from typing import Optional, List + +import torch +import torch.nn.functional as F +from torch import nn, Tensor + +import IPython + +e = IPython.embed + + +class Transformer(nn.Module): + + def __init__(self, + d_model=512, + nhead=8, + num_encoder_layers=6, + num_decoder_layers=6, + dim_feedforward=2048, + dropout=0.1, + activation="relu", + normalize_before=False, + return_intermediate_dec=False): + super().__init__() + + encoder_layer = TransformerEncoderLayer(d_model, nhead, dim_feedforward, dropout, activation, normalize_before) + encoder_norm = nn.LayerNorm(d_model) if normalize_before else None + self.encoder = TransformerEncoder(encoder_layer, num_encoder_layers, encoder_norm) + + decoder_layer = TransformerDecoderLayer(d_model, nhead, dim_feedforward, dropout, activation, normalize_before) + decoder_norm = nn.LayerNorm(d_model) + self.decoder = TransformerDecoder(decoder_layer, + num_decoder_layers, + decoder_norm, + return_intermediate=return_intermediate_dec) + + self._reset_parameters() + + self.d_model = d_model + self.nhead = nhead + + def _reset_parameters(self): + for p in self.parameters(): + if p.dim() > 1: + nn.init.xavier_uniform_(p) + + def forward(self, + src, + mask, + query_embed, + pos_embed, + latent_input=None, + proprio_input=None, + additional_pos_embed=None): + # TODO flatten only when input has H and W + if len(src.shape) == 4: # has H and W + # flatten NxCxHxW to HWxNxC + bs, c, h, w = src.shape + src = src.flatten(2).permute(2, 0, 1) + pos_embed = pos_embed.flatten(2).permute(2, 0, 1).repeat(1, bs, 1) + query_embed = query_embed.unsqueeze(1).repeat(1, bs, 1) + # mask = mask.flatten(1) + + additional_pos_embed = additional_pos_embed.unsqueeze(1).repeat(1, bs, 1) # seq, bs, dim + pos_embed = torch.cat([additional_pos_embed, pos_embed], axis=0) + + addition_input = torch.stack([latent_input, proprio_input], axis=0) + src = torch.cat([addition_input, src], axis=0) + else: + assert len(src.shape) == 3 + # flatten NxHWxC to HWxNxC + bs, hw, c = src.shape + src = src.permute(1, 0, 2) + pos_embed = pos_embed.unsqueeze(1).repeat(1, bs, 1) + query_embed = query_embed.unsqueeze(1).repeat(1, bs, 1) + + tgt = torch.zeros_like(query_embed) + memory = self.encoder(src, src_key_padding_mask=mask, pos=pos_embed) + hs = self.decoder(tgt, memory, memory_key_padding_mask=mask, pos=pos_embed, query_pos=query_embed) + hs = hs.transpose(1, 2) + return hs + + +class TransformerEncoder(nn.Module): + + def __init__(self, encoder_layer, num_layers, norm=None): + super().__init__() + self.layers = _get_clones(encoder_layer, num_layers) + self.num_layers = num_layers + self.norm = norm + + def forward(self, + src, + mask: Optional[Tensor] = None, + src_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None): + output = src + + for layer in self.layers: + output = layer(output, src_mask=mask, src_key_padding_mask=src_key_padding_mask, pos=pos) + + if self.norm is not None: + output = self.norm(output) + + return output + + +class TransformerDecoder(nn.Module): + + def __init__(self, decoder_layer, num_layers, norm=None, return_intermediate=False): + super().__init__() + self.layers = _get_clones(decoder_layer, num_layers) + self.num_layers = num_layers + self.norm = norm + self.return_intermediate = return_intermediate + + def forward(self, + tgt, + memory, + tgt_mask: Optional[Tensor] = None, + memory_mask: Optional[Tensor] = None, + tgt_key_padding_mask: Optional[Tensor] = None, + memory_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None): + output = tgt + + intermediate = [] + + for layer in self.layers: + output = layer(output, + memory, + tgt_mask=tgt_mask, + memory_mask=memory_mask, + tgt_key_padding_mask=tgt_key_padding_mask, + memory_key_padding_mask=memory_key_padding_mask, + pos=pos, + query_pos=query_pos) + if self.return_intermediate: + intermediate.append(self.norm(output)) + + if self.norm is not None: + output = self.norm(output) + if self.return_intermediate: + intermediate.pop() + intermediate.append(output) + + if self.return_intermediate: + return torch.stack(intermediate) + + return output.unsqueeze(0) + + +class TransformerEncoderLayer(nn.Module): + + def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1, activation="relu", normalize_before=False): + super().__init__() + self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout) + # Implementation of Feedforward model + self.linear1 = nn.Linear(d_model, dim_feedforward) + self.dropout = nn.Dropout(dropout) + self.linear2 = nn.Linear(dim_feedforward, d_model) + + self.norm1 = nn.LayerNorm(d_model) + self.norm2 = nn.LayerNorm(d_model) + self.dropout1 = nn.Dropout(dropout) + self.dropout2 = nn.Dropout(dropout) + + self.activation = _get_activation_fn(activation) + self.normalize_before = normalize_before + + def with_pos_embed(self, tensor, pos: Optional[Tensor]): + return tensor if pos is None else tensor + pos + + def forward_post(self, + src, + src_mask: Optional[Tensor] = None, + src_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None): + q = k = self.with_pos_embed(src, pos) + src2 = self.self_attn(q, k, value=src, attn_mask=src_mask, key_padding_mask=src_key_padding_mask)[0] + src = src + self.dropout1(src2) + src = self.norm1(src) + src2 = self.linear2(self.dropout(self.activation(self.linear1(src)))) + src = src + self.dropout2(src2) + src = self.norm2(src) + return src + + def forward_pre(self, + src, + src_mask: Optional[Tensor] = None, + src_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None): + src2 = self.norm1(src) + q = k = self.with_pos_embed(src2, pos) + src2 = self.self_attn(q, k, value=src2, attn_mask=src_mask, key_padding_mask=src_key_padding_mask)[0] + src = src + self.dropout1(src2) + src2 = self.norm2(src) + src2 = self.linear2(self.dropout(self.activation(self.linear1(src2)))) + src = src + self.dropout2(src2) + return src + + def forward(self, + src, + src_mask: Optional[Tensor] = None, + src_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None): + if self.normalize_before: + return self.forward_pre(src, src_mask, src_key_padding_mask, pos) + return self.forward_post(src, src_mask, src_key_padding_mask, pos) + + +class TransformerDecoderLayer(nn.Module): + + def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1, activation="relu", normalize_before=False): + super().__init__() + self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout) + self.multihead_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout) + # Implementation of Feedforward model + self.linear1 = nn.Linear(d_model, dim_feedforward) + self.dropout = nn.Dropout(dropout) + self.linear2 = nn.Linear(dim_feedforward, d_model) + + self.norm1 = nn.LayerNorm(d_model) + self.norm2 = nn.LayerNorm(d_model) + self.norm3 = nn.LayerNorm(d_model) + self.dropout1 = nn.Dropout(dropout) + self.dropout2 = nn.Dropout(dropout) + self.dropout3 = nn.Dropout(dropout) + + self.activation = _get_activation_fn(activation) + self.normalize_before = normalize_before + + def with_pos_embed(self, tensor, pos: Optional[Tensor]): + return tensor if pos is None else tensor + pos + + def forward_post(self, + tgt, + memory, + tgt_mask: Optional[Tensor] = None, + memory_mask: Optional[Tensor] = None, + tgt_key_padding_mask: Optional[Tensor] = None, + memory_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None): + q = k = self.with_pos_embed(tgt, query_pos) + tgt2 = self.self_attn(q, k, value=tgt, attn_mask=tgt_mask, key_padding_mask=tgt_key_padding_mask)[0] + tgt = tgt + self.dropout1(tgt2) + tgt = self.norm1(tgt) + tgt2 = self.multihead_attn(query=self.with_pos_embed(tgt, query_pos), + key=self.with_pos_embed(memory, pos), + value=memory, + attn_mask=memory_mask, + key_padding_mask=memory_key_padding_mask)[0] + tgt = tgt + self.dropout2(tgt2) + tgt = self.norm2(tgt) + tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt)))) + tgt = tgt + self.dropout3(tgt2) + tgt = self.norm3(tgt) + return tgt + + def forward_pre(self, + tgt, + memory, + tgt_mask: Optional[Tensor] = None, + memory_mask: Optional[Tensor] = None, + tgt_key_padding_mask: Optional[Tensor] = None, + memory_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None): + tgt2 = self.norm1(tgt) + q = k = self.with_pos_embed(tgt2, query_pos) + tgt2 = self.self_attn(q, k, value=tgt2, attn_mask=tgt_mask, key_padding_mask=tgt_key_padding_mask)[0] + tgt = tgt + self.dropout1(tgt2) + tgt2 = self.norm2(tgt) + tgt2 = self.multihead_attn(query=self.with_pos_embed(tgt2, query_pos), + key=self.with_pos_embed(memory, pos), + value=memory, + attn_mask=memory_mask, + key_padding_mask=memory_key_padding_mask)[0] + tgt = tgt + self.dropout2(tgt2) + tgt2 = self.norm3(tgt) + tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt2)))) + tgt = tgt + self.dropout3(tgt2) + return tgt + + def forward(self, + tgt, + memory, + tgt_mask: Optional[Tensor] = None, + memory_mask: Optional[Tensor] = None, + tgt_key_padding_mask: Optional[Tensor] = None, + memory_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None): + if self.normalize_before: + return self.forward_pre(tgt, memory, tgt_mask, memory_mask, tgt_key_padding_mask, memory_key_padding_mask, + pos, query_pos) + return self.forward_post(tgt, memory, tgt_mask, memory_mask, tgt_key_padding_mask, memory_key_padding_mask, pos, + query_pos) + + +def _get_clones(module, N): + return nn.ModuleList([copy.deepcopy(module) for i in range(N)]) + + +def build_transformer(args): + return Transformer( + d_model=args.hidden_dim, + dropout=args.dropout, + nhead=args.nheads, + dim_feedforward=args.dim_feedforward, + num_encoder_layers=args.enc_layers, + num_decoder_layers=args.dec_layers, + normalize_before=args.pre_norm, + return_intermediate_dec=True, + ) + + +def _get_activation_fn(activation): + """Return an activation function given a string""" + if activation == "relu": + return F.relu + if activation == "gelu": + return F.gelu + if activation == "glu": + return F.glu + raise RuntimeError(F"activation should be relu/gelu, not {activation}.") diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/setup.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/setup.py new file mode 100644 index 00000000..076bc4aa --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/setup.py @@ -0,0 +1,10 @@ +from distutils.core import setup +from setuptools import find_packages + +setup( + name="detr", + version="0.0.0", + packages=find_packages(), + license="MIT License", + long_description=open("README.md").read(), +) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/util/__init__.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/util/__init__.py new file mode 100644 index 00000000..168f9979 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/util/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/util/box_ops.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/util/box_ops.py new file mode 100644 index 00000000..d8544b3d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/util/box_ops.py @@ -0,0 +1,86 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Utilities for bounding box manipulation and GIoU. +""" +import torch +from torchvision.ops.boxes import box_area + + +def box_cxcywh_to_xyxy(x): + x_c, y_c, w, h = x.unbind(-1) + b = [(x_c - 0.5 * w), (y_c - 0.5 * h), (x_c + 0.5 * w), (y_c + 0.5 * h)] + return torch.stack(b, dim=-1) + + +def box_xyxy_to_cxcywh(x): + x0, y0, x1, y1 = x.unbind(-1) + b = [(x0 + x1) / 2, (y0 + y1) / 2, (x1 - x0), (y1 - y0)] + return torch.stack(b, dim=-1) + + +# modified from torchvision to also return the union +def box_iou(boxes1, boxes2): + area1 = box_area(boxes1) + area2 = box_area(boxes2) + + lt = torch.max(boxes1[:, None, :2], boxes2[:, :2]) # [N,M,2] + rb = torch.min(boxes1[:, None, 2:], boxes2[:, 2:]) # [N,M,2] + + wh = (rb - lt).clamp(min=0) # [N,M,2] + inter = wh[:, :, 0] * wh[:, :, 1] # [N,M] + + union = area1[:, None] + area2 - inter + + iou = inter / union + return iou, union + + +def generalized_box_iou(boxes1, boxes2): + """ + Generalized IoU from https://giou.stanford.edu/ + + The boxes should be in [x0, y0, x1, y1] format + + Returns a [N, M] pairwise matrix, where N = len(boxes1) + and M = len(boxes2) + """ + # degenerate boxes gives inf / nan results + # so do an early check + assert (boxes1[:, 2:] >= boxes1[:, :2]).all() + assert (boxes2[:, 2:] >= boxes2[:, :2]).all() + iou, union = box_iou(boxes1, boxes2) + + lt = torch.min(boxes1[:, None, :2], boxes2[:, :2]) + rb = torch.max(boxes1[:, None, 2:], boxes2[:, 2:]) + + wh = (rb - lt).clamp(min=0) # [N,M,2] + area = wh[:, :, 0] * wh[:, :, 1] + + return iou - (area - union) / area + + +def masks_to_boxes(masks): + """Compute the bounding boxes around the provided masks + + The masks should be in format [N, H, W] where N is the number of masks, (H, W) are the spatial dimensions. + + Returns a [N, 4] tensors, with the boxes in xyxy format + """ + if masks.numel() == 0: + return torch.zeros((0, 4), device=masks.device) + + h, w = masks.shape[-2:] + + y = torch.arange(0, h, dtype=torch.float) + x = torch.arange(0, w, dtype=torch.float) + y, x = torch.meshgrid(y, x) + + x_mask = masks * x.unsqueeze(0) + x_max = x_mask.flatten(1).max(-1)[0] + x_min = x_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0] + + y_mask = masks * y.unsqueeze(0) + y_max = y_mask.flatten(1).max(-1)[0] + y_min = y_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0] + + return torch.stack([x_min, y_min, x_max, y_max], 1) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/util/misc.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/util/misc.py new file mode 100644 index 00000000..07864352 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/util/misc.py @@ -0,0 +1,481 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Misc functions, including distributed helpers. + +Mostly copy-paste from torchvision references. +""" +import os +import subprocess +import time +from collections import defaultdict, deque +import datetime +import pickle +from packaging import version +from typing import Optional, List + +import torch +import torch.distributed as dist +from torch import Tensor + +# needed due to empty tensor bug in pytorch and torchvision 0.5 +import torchvision + +if version.parse(torchvision.__version__) < version.parse("0.7"): + from torchvision.ops import _new_empty_tensor + from torchvision.ops.misc import _output_size + + +class SmoothedValue(object): + """Track a series of values and provide access to smoothed values over a + window or the global series average. + """ + + def __init__(self, window_size=20, fmt=None): + if fmt is None: + fmt = "{median:.4f} ({global_avg:.4f})" + self.deque = deque(maxlen=window_size) + self.total = 0.0 + self.count = 0 + self.fmt = fmt + + def update(self, value, n=1): + self.deque.append(value) + self.count += n + self.total += value * n + + def synchronize_between_processes(self): + """ + Warning: does not synchronize the deque! + """ + if not is_dist_avail_and_initialized(): + return + t = torch.tensor([self.count, self.total], dtype=torch.float64, device="cuda") + dist.barrier() + dist.all_reduce(t) + t = t.tolist() + self.count = int(t[0]) + self.total = t[1] + + @property + def median(self): + d = torch.tensor(list(self.deque)) + return d.median().item() + + @property + def avg(self): + d = torch.tensor(list(self.deque), dtype=torch.float32) + return d.mean().item() + + @property + def global_avg(self): + return self.total / self.count + + @property + def max(self): + return max(self.deque) + + @property + def value(self): + return self.deque[-1] + + def __str__(self): + return self.fmt.format( + median=self.median, + avg=self.avg, + global_avg=self.global_avg, + max=self.max, + value=self.value, + ) + + +def all_gather(data): + """ + Run all_gather on arbitrary picklable data (not necessarily tensors) + Args: + data: any picklable object + Returns: + list[data]: list of data gathered from each rank + """ + world_size = get_world_size() + if world_size == 1: + return [data] + + # serialized to a Tensor + buffer = pickle.dumps(data) + storage = torch.ByteStorage.from_buffer(buffer) + tensor = torch.ByteTensor(storage).to("cuda") + + # obtain Tensor size of each rank + local_size = torch.tensor([tensor.numel()], device="cuda") + size_list = [torch.tensor([0], device="cuda") for _ in range(world_size)] + dist.all_gather(size_list, local_size) + size_list = [int(size.item()) for size in size_list] + max_size = max(size_list) + + # receiving Tensor from all ranks + # we pad the tensor because torch all_gather does not support + # gathering tensors of different shapes + tensor_list = [] + for _ in size_list: + tensor_list.append(torch.empty((max_size, ), dtype=torch.uint8, device="cuda")) + if local_size != max_size: + padding = torch.empty(size=(max_size - local_size, ), dtype=torch.uint8, device="cuda") + tensor = torch.cat((tensor, padding), dim=0) + dist.all_gather(tensor_list, tensor) + + data_list = [] + for size, tensor in zip(size_list, tensor_list): + buffer = tensor.cpu().numpy().tobytes()[:size] + data_list.append(pickle.loads(buffer)) + + return data_list + + +def reduce_dict(input_dict, average=True): + """ + Args: + input_dict (dict): all the values will be reduced + average (bool): whether to do average or sum + Reduce the values in the dictionary from all processes so that all processes + have the averaged results. Returns a dict with the same fields as + input_dict, after reduction. + """ + world_size = get_world_size() + if world_size < 2: + return input_dict + with torch.no_grad(): + names = [] + values = [] + # sort the keys so that they are consistent across processes + for k in sorted(input_dict.keys()): + names.append(k) + values.append(input_dict[k]) + values = torch.stack(values, dim=0) + dist.all_reduce(values) + if average: + values /= world_size + reduced_dict = {k: v for k, v in zip(names, values)} + return reduced_dict + + +class MetricLogger(object): + + def __init__(self, delimiter="\t"): + self.meters = defaultdict(SmoothedValue) + self.delimiter = delimiter + + def update(self, **kwargs): + for k, v in kwargs.items(): + if isinstance(v, torch.Tensor): + v = v.item() + assert isinstance(v, (float, int)) + self.meters[k].update(v) + + def __getattr__(self, attr): + if attr in self.meters: + return self.meters[attr] + if attr in self.__dict__: + return self.__dict__[attr] + raise AttributeError("'{}' object has no attribute '{}'".format(type(self).__name__, attr)) + + def __str__(self): + loss_str = [] + for name, meter in self.meters.items(): + loss_str.append("{}: {}".format(name, str(meter))) + return self.delimiter.join(loss_str) + + def synchronize_between_processes(self): + for meter in self.meters.values(): + meter.synchronize_between_processes() + + def add_meter(self, name, meter): + self.meters[name] = meter + + def log_every(self, iterable, print_freq, header=None): + i = 0 + if not header: + header = "" + start_time = time.time() + end = time.time() + iter_time = SmoothedValue(fmt="{avg:.4f}") + data_time = SmoothedValue(fmt="{avg:.4f}") + space_fmt = ":" + str(len(str(len(iterable)))) + "d" + if torch.cuda.is_available(): + log_msg = self.delimiter.join([ + header, + "[{0" + space_fmt + "}/{1}]", + "eta: {eta}", + "{meters}", + "time: {time}", + "data: {data}", + "max mem: {memory:.0f}", + ]) + else: + log_msg = self.delimiter.join([ + header, + "[{0" + space_fmt + "}/{1}]", + "eta: {eta}", + "{meters}", + "time: {time}", + "data: {data}", + ]) + MB = 1024.0 * 1024.0 + for obj in iterable: + data_time.update(time.time() - end) + yield obj + iter_time.update(time.time() - end) + if i % print_freq == 0 or i == len(iterable) - 1: + eta_seconds = iter_time.global_avg * (len(iterable) - i) + eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) + if torch.cuda.is_available(): + print( + log_msg.format( + i, + len(iterable), + eta=eta_string, + meters=str(self), + time=str(iter_time), + data=str(data_time), + memory=torch.cuda.max_memory_allocated() / MB, + )) + else: + print( + log_msg.format( + i, + len(iterable), + eta=eta_string, + meters=str(self), + time=str(iter_time), + data=str(data_time), + )) + i += 1 + end = time.time() + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print("{} Total time: {} ({:.4f} s / it)".format(header, total_time_str, total_time / len(iterable))) + + +def get_sha(): + cwd = os.path.dirname(os.path.abspath(__file__)) + + def _run(command): + return subprocess.check_output(command, cwd=cwd).decode("ascii").strip() + + sha = "N/A" + diff = "clean" + branch = "N/A" + try: + sha = _run(["git", "rev-parse", "HEAD"]) + subprocess.check_output(["git", "diff"], cwd=cwd) + diff = _run(["git", "diff-index", "HEAD"]) + diff = "has uncommited changes" if diff else "clean" + branch = _run(["git", "rev-parse", "--abbrev-ref", "HEAD"]) + except Exception: + pass + message = f"sha: {sha}, status: {diff}, branch: {branch}" + return message + + +def collate_fn(batch): + batch = list(zip(*batch)) + batch[0] = nested_tensor_from_tensor_list(batch[0]) + return tuple(batch) + + +def _max_by_axis(the_list): + # type: (List[List[int]]) -> List[int] + maxes = the_list[0] + for sublist in the_list[1:]: + for index, item in enumerate(sublist): + maxes[index] = max(maxes[index], item) + return maxes + + +class NestedTensor(object): + + def __init__(self, tensors, mask: Optional[Tensor]): + self.tensors = tensors + self.mask = mask + + def to(self, device): + # type: (Device) -> NestedTensor # noqa + cast_tensor = self.tensors.to(device) + mask = self.mask + if mask is not None: + assert mask is not None + cast_mask = mask.to(device) + else: + cast_mask = None + return NestedTensor(cast_tensor, cast_mask) + + def decompose(self): + return self.tensors, self.mask + + def __repr__(self): + return str(self.tensors) + + +def nested_tensor_from_tensor_list(tensor_list: List[Tensor]): + # TODO make this more general + if tensor_list[0].ndim == 3: + if torchvision._is_tracing(): + # nested_tensor_from_tensor_list() does not export well to ONNX + # call _onnx_nested_tensor_from_tensor_list() instead + return _onnx_nested_tensor_from_tensor_list(tensor_list) + + # TODO make it support different-sized images + max_size = _max_by_axis([list(img.shape) for img in tensor_list]) + # min_size = tuple(min(s) for s in zip(*[img.shape for img in tensor_list])) + batch_shape = [len(tensor_list)] + max_size + b, c, h, w = batch_shape + dtype = tensor_list[0].dtype + device = tensor_list[0].device + tensor = torch.zeros(batch_shape, dtype=dtype, device=device) + mask = torch.ones((b, h, w), dtype=torch.bool, device=device) + for img, pad_img, m in zip(tensor_list, tensor, mask): + pad_img[:img.shape[0], :img.shape[1], :img.shape[2]].copy_(img) + m[:img.shape[1], :img.shape[2]] = False + else: + raise ValueError("not supported") + return NestedTensor(tensor, mask) + + +# _onnx_nested_tensor_from_tensor_list() is an implementation of +# nested_tensor_from_tensor_list() that is supported by ONNX tracing. +@torch.jit.unused +def _onnx_nested_tensor_from_tensor_list(tensor_list: List[Tensor]) -> NestedTensor: + max_size = [] + for i in range(tensor_list[0].dim()): + max_size_i = torch.max(torch.stack([img.shape[i] for img in tensor_list]).to(torch.float32)).to(torch.int64) + max_size.append(max_size_i) + max_size = tuple(max_size) + + # work around for + # pad_img[: img.shape[0], : img.shape[1], : img.shape[2]].copy_(img) + # m[: img.shape[1], :img.shape[2]] = False + # which is not yet supported in onnx + padded_imgs = [] + padded_masks = [] + for img in tensor_list: + padding = [(s1 - s2) for s1, s2 in zip(max_size, tuple(img.shape))] + padded_img = torch.nn.functional.pad(img, (0, padding[2], 0, padding[1], 0, padding[0])) + padded_imgs.append(padded_img) + + m = torch.zeros_like(img[0], dtype=torch.int, device=img.device) + padded_mask = torch.nn.functional.pad(m, (0, padding[2], 0, padding[1]), "constant", 1) + padded_masks.append(padded_mask.to(torch.bool)) + + tensor = torch.stack(padded_imgs) + mask = torch.stack(padded_masks) + + return NestedTensor(tensor, mask=mask) + + +def setup_for_distributed(is_master): + """ + This function disables printing when not in master process + """ + import builtins as __builtin__ + + builtin_print = __builtin__.print + + def print(*args, **kwargs): + force = kwargs.pop("force", False) + if is_master or force: + builtin_print(*args, **kwargs) + + __builtin__.print = print + + +def is_dist_avail_and_initialized(): + if not dist.is_available(): + return False + if not dist.is_initialized(): + return False + return True + + +def get_world_size(): + if not is_dist_avail_and_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank(): + if not is_dist_avail_and_initialized(): + return 0 + return dist.get_rank() + + +def is_main_process(): + return get_rank() == 0 + + +def save_on_master(*args, **kwargs): + if is_main_process(): + torch.save(*args, **kwargs) + + +def init_distributed_mode(args): + if "RANK" in os.environ and "WORLD_SIZE" in os.environ: + args.rank = int(os.environ["RANK"]) + args.world_size = int(os.environ["WORLD_SIZE"]) + args.gpu = int(os.environ["LOCAL_RANK"]) + elif "SLURM_PROCID" in os.environ: + args.rank = int(os.environ["SLURM_PROCID"]) + args.gpu = args.rank % torch.cuda.device_count() + else: + print("Not using distributed mode") + args.distributed = False + return + + args.distributed = True + + torch.cuda.set_device(args.gpu) + args.dist_backend = "nccl" + print("| distributed init (rank {}): {}".format(args.rank, args.dist_url), flush=True) + torch.distributed.init_process_group( + backend=args.dist_backend, + init_method=args.dist_url, + world_size=args.world_size, + rank=args.rank, + ) + torch.distributed.barrier() + setup_for_distributed(args.rank == 0) + + +@torch.no_grad() +def accuracy(output, target, topk=(1, )): + """Computes the precision@k for the specified values of k""" + if target.numel() == 0: + return [torch.zeros([], device=output.device)] + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +def interpolate(input, size=None, scale_factor=None, mode="nearest", align_corners=None): + # type: (Tensor, Optional[List[int]], Optional[float], str, Optional[bool]) -> Tensor + """ + Equivalent to nn.functional.interpolate, but with support for empty batch sizes. + This will eventually be supported natively by PyTorch, and this + class can go away. + """ + if version.parse(torchvision.__version__) < version.parse("0.7"): + if input.numel() > 0: + return torch.nn.functional.interpolate(input, size, scale_factor, mode, align_corners) + + output_shape = _output_size(2, input, size, scale_factor) + output_shape = list(input.shape[:-2]) + list(output_shape) + return _new_empty_tensor(input, output_shape) + else: + return torchvision.ops.misc.interpolate(input, size, scale_factor, mode, align_corners) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/util/plot_utils.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/util/plot_utils.py new file mode 100644 index 00000000..9c1a15e6 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/detr/util/plot_utils.py @@ -0,0 +1,110 @@ +""" +Plotting utilities to visualize training logs. +""" + +import torch +import pandas as pd +import numpy as np +import seaborn as sns +import matplotlib.pyplot as plt + +from pathlib import Path, PurePath + + +def plot_logs( + logs, + fields=("class_error", "loss_bbox_unscaled", "mAP"), + ewm_col=0, + log_name="log.txt", +): + """ + Function to plot specific fields from training log(s). Plots both training and test results. + + :: Inputs - logs = list containing Path objects, each pointing to individual dir with a log file + - fields = which results to plot from each log file - plots both training and test for each field. + - ewm_col = optional, which column to use as the exponential weighted smoothing of the plots + - log_name = optional, name of log file if different than default 'log.txt'. + + :: Outputs - matplotlib plots of results in fields, color coded for each log file. + - solid lines are training results, dashed lines are test results. + + """ + func_name = "plot_utils.py::plot_logs" + + # verify logs is a list of Paths (list[Paths]) or single Pathlib object Path, + # convert single Path to list to avoid 'not iterable' error + + if not isinstance(logs, list): + if isinstance(logs, PurePath): + logs = [logs] + print(f"{func_name} info: logs param expects a list argument, converted to list[Path].") + else: + raise ValueError(f"{func_name} - invalid argument for logs parameter.\n \ + Expect list[Path] or single Path obj, received {type(logs)}") + + # Quality checks - verify valid dir(s), that every item in list is Path object, and that log_name exists in each dir + for i, dir in enumerate(logs): + if not isinstance(dir, PurePath): + raise ValueError(f"{func_name} - non-Path object in logs argument of {type(dir)}: \n{dir}") + if not dir.exists(): + raise ValueError(f"{func_name} - invalid directory in logs argument:\n{dir}") + # verify log_name exists + fn = Path(dir / log_name) + if not fn.exists(): + print(f"-> missing {log_name}. Have you gotten to Epoch 1 in training?") + print(f"--> full path of missing log file: {fn}") + return + + # load log file(s) and plot + dfs = [pd.read_json(Path(p) / log_name, lines=True) for p in logs] + + fig, axs = plt.subplots(ncols=len(fields), figsize=(16, 5)) + + for df, color in zip(dfs, sns.color_palette(n_colors=len(logs))): + for j, field in enumerate(fields): + if field == "mAP": + coco_eval = (pd.DataFrame(np.stack(df.test_coco_eval_bbox.dropna().values)[:, + 1]).ewm(com=ewm_col).mean()) + axs[j].plot(coco_eval, c=color) + else: + df.interpolate().ewm(com=ewm_col).mean().plot( + y=[f"train_{field}", f"test_{field}"], + ax=axs[j], + color=[color] * 2, + style=["-", "--"], + ) + for ax, field in zip(axs, fields): + ax.legend([Path(p).name for p in logs]) + ax.set_title(field) + + +def plot_precision_recall(files, naming_scheme="iter"): + if naming_scheme == "exp_id": + # name becomes exp_id + names = [f.parts[-3] for f in files] + elif naming_scheme == "iter": + names = [f.stem for f in files] + else: + raise ValueError(f"not supported {naming_scheme}") + fig, axs = plt.subplots(ncols=2, figsize=(16, 5)) + for f, color, name in zip(files, sns.color_palette("Blues", n_colors=len(files)), names): + data = torch.load(f) + # precision is n_iou, n_points, n_cat, n_area, max_det + precision = data["precision"] + recall = data["params"].recThrs + scores = data["scores"] + # take precision for all classes, all areas and 100 detections + precision = precision[0, :, :, 0, -1].mean(1) + scores = scores[0, :, :, 0, -1].mean(1) + prec = precision.mean() + rec = data["recall"][0, :, 0, -1].mean() + print(f"{naming_scheme} {name}: mAP@50={prec * 100: 05.1f}, " + f"score={scores.mean():0.3f}, " + + f"f1={2 * prec * rec / (prec + rec + 1e-8):0.3f}") + axs[0].plot(recall, precision, c=color) + axs[1].plot(recall, scores, c=color) + + axs[0].set_title("Precision / Recall") + axs[0].legend(names) + axs[1].set_title("Scores / Recall") + axs[1].legend(names) + return fig, axs diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/ee_sim_env.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/ee_sim_env.py new file mode 100644 index 00000000..a701abac --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/ee_sim_env.py @@ -0,0 +1,295 @@ +import numpy as np +import collections +import os + +from constants import DT, XML_DIR, START_ARM_POSE +from constants import PUPPET_GRIPPER_POSITION_CLOSE +from constants import PUPPET_GRIPPER_POSITION_UNNORMALIZE_FN +from constants import PUPPET_GRIPPER_POSITION_NORMALIZE_FN +from constants import PUPPET_GRIPPER_VELOCITY_NORMALIZE_FN + +from utils import sample_box_pose, sample_insertion_pose +from dm_control import mujoco +from dm_control.rl import control +from dm_control.suite import base + +import IPython + +e = IPython.embed + + +def make_ee_sim_env(task_name): + """ + Environment for simulated robot bi-manual manipulation, with end-effector control. + Action space: [left_arm_pose (7), # position and quaternion for end effector + left_gripper_positions (1), # normalized gripper position (0: close, 1: open) + right_arm_pose (7), # position and quaternion for end effector + right_gripper_positions (1),] # normalized gripper position (0: close, 1: open) + + Observation space: {"qpos": Concat[ left_arm_qpos (6), # absolute joint position + left_gripper_position (1), # normalized gripper position (0: close, 1: open) + right_arm_qpos (6), # absolute joint position + right_gripper_qpos (1)] # normalized gripper position (0: close, 1: open) + "qvel": Concat[ left_arm_qvel (6), # absolute joint velocity (rad) + left_gripper_velocity (1), # normalized gripper velocity (pos: opening, neg: closing) + right_arm_qvel (6), # absolute joint velocity (rad) + right_gripper_qvel (1)] # normalized gripper velocity (pos: opening, neg: closing) + "images": {"main": (480x640x3)} # h, w, c, dtype='uint8' + """ + if "sim_transfer_cube" in task_name: + xml_path = os.path.join(XML_DIR, f"bimanual_viperx_ee_transfer_cube.xml") + physics = mujoco.Physics.from_xml_path(xml_path) + task = TransferCubeEETask(random=False) + env = control.Environment( + physics, + task, + time_limit=20, + control_timestep=DT, + n_sub_steps=None, + flat_observation=False, + ) + elif "sim_insertion" in task_name: + xml_path = os.path.join(XML_DIR, f"bimanual_viperx_ee_insertion.xml") + physics = mujoco.Physics.from_xml_path(xml_path) + task = InsertionEETask(random=False) + env = control.Environment( + physics, + task, + time_limit=20, + control_timestep=DT, + n_sub_steps=None, + flat_observation=False, + ) + else: + raise NotImplementedError + return env + + +class BimanualViperXEETask(base.Task): + + def __init__(self, random=None): + super().__init__(random=random) + + def before_step(self, action, physics): + a_len = len(action) // 2 + action_left = action[:a_len] + action_right = action[a_len:] + + # set mocap position and quat + # left + np.copyto(physics.data.mocap_pos[0], action_left[:3]) + np.copyto(physics.data.mocap_quat[0], action_left[3:7]) + # right + np.copyto(physics.data.mocap_pos[1], action_right[:3]) + np.copyto(physics.data.mocap_quat[1], action_right[3:7]) + + # set gripper + g_left_ctrl = PUPPET_GRIPPER_POSITION_UNNORMALIZE_FN(action_left[7]) + g_right_ctrl = PUPPET_GRIPPER_POSITION_UNNORMALIZE_FN(action_right[7]) + np.copyto( + physics.data.ctrl, + np.array([g_left_ctrl, -g_left_ctrl, g_right_ctrl, -g_right_ctrl]), + ) + + def initialize_robots(self, physics): + # reset joint position + physics.named.data.qpos[:16] = START_ARM_POSE + + # reset mocap to align with end effector + # to obtain these numbers: + # (1) make an ee_sim env and reset to the same start_pose + # (2) get env._physics.named.data.xpos['vx300s_left/gripper_link'] + # get env._physics.named.data.xquat['vx300s_left/gripper_link'] + # repeat the same for right side + np.copyto(physics.data.mocap_pos[0], [-0.31718881, 0.5, 0.29525084]) + np.copyto(physics.data.mocap_quat[0], [1, 0, 0, 0]) + # right + np.copyto(physics.data.mocap_pos[1], np.array([0.31718881, 0.49999888, 0.29525084])) + np.copyto(physics.data.mocap_quat[1], [1, 0, 0, 0]) + + # reset gripper control + close_gripper_control = np.array([ + PUPPET_GRIPPER_POSITION_CLOSE, + -PUPPET_GRIPPER_POSITION_CLOSE, + PUPPET_GRIPPER_POSITION_CLOSE, + -PUPPET_GRIPPER_POSITION_CLOSE, + ]) + np.copyto(physics.data.ctrl, close_gripper_control) + + def initialize_episode(self, physics): + """Sets the state of the environment at the start of each episode.""" + super().initialize_episode(physics) + + @staticmethod + def get_qpos(physics): + qpos_raw = physics.data.qpos.copy() + left_qpos_raw = qpos_raw[:8] + right_qpos_raw = qpos_raw[8:16] + left_arm_qpos = left_qpos_raw[:6] + right_arm_qpos = right_qpos_raw[:6] + left_gripper_qpos = [PUPPET_GRIPPER_POSITION_NORMALIZE_FN(left_qpos_raw[6])] + right_gripper_qpos = [PUPPET_GRIPPER_POSITION_NORMALIZE_FN(right_qpos_raw[6])] + return np.concatenate([left_arm_qpos, left_gripper_qpos, right_arm_qpos, right_gripper_qpos]) + + @staticmethod + def get_qvel(physics): + qvel_raw = physics.data.qvel.copy() + left_qvel_raw = qvel_raw[:8] + right_qvel_raw = qvel_raw[8:16] + left_arm_qvel = left_qvel_raw[:6] + right_arm_qvel = right_qvel_raw[:6] + left_gripper_qvel = [PUPPET_GRIPPER_VELOCITY_NORMALIZE_FN(left_qvel_raw[6])] + right_gripper_qvel = [PUPPET_GRIPPER_VELOCITY_NORMALIZE_FN(right_qvel_raw[6])] + return np.concatenate([left_arm_qvel, left_gripper_qvel, right_arm_qvel, right_gripper_qvel]) + + @staticmethod + def get_env_state(physics): + raise NotImplementedError + + def get_observation(self, physics): + # note: it is important to do .copy() + obs = collections.OrderedDict() + obs["qpos"] = self.get_qpos(physics) + obs["qvel"] = self.get_qvel(physics) + obs["env_state"] = self.get_env_state(physics) + obs["images"] = dict() + obs["images"]["top"] = physics.render(height=480, width=640, camera_id="top") + obs["images"]["angle"] = physics.render(height=480, width=640, camera_id="angle") + obs["images"]["vis"] = physics.render(height=480, width=640, camera_id="front_close") + # used in scripted policy to obtain starting pose + obs["mocap_pose_left"] = np.concatenate([physics.data.mocap_pos[0], physics.data.mocap_quat[0]]).copy() + obs["mocap_pose_right"] = np.concatenate([physics.data.mocap_pos[1], physics.data.mocap_quat[1]]).copy() + + # used when replaying joint trajectory + obs["gripper_ctrl"] = physics.data.ctrl.copy() + return obs + + def get_reward(self, physics): + raise NotImplementedError + + +class TransferCubeEETask(BimanualViperXEETask): + + def __init__(self, random=None): + super().__init__(random=random) + self.max_reward = 4 + + def initialize_episode(self, physics): + """Sets the state of the environment at the start of each episode.""" + self.initialize_robots(physics) + # randomize box position + cube_pose = sample_box_pose() + box_start_idx = physics.model.name2id("red_box_joint", "joint") + np.copyto(physics.data.qpos[box_start_idx:box_start_idx + 7], cube_pose) + # print(f"randomized cube position to {cube_position}") + + super().initialize_episode(physics) + + @staticmethod + def get_env_state(physics): + env_state = physics.data.qpos.copy()[16:] + return env_state + + def get_reward(self, physics): + # return whether left gripper is holding the box + all_contact_pairs = [] + for i_contact in range(physics.data.ncon): + id_geom_1 = physics.data.contact[i_contact].geom1 + id_geom_2 = physics.data.contact[i_contact].geom2 + name_geom_1 = physics.model.id2name(id_geom_1, "geom") + name_geom_2 = physics.model.id2name(id_geom_2, "geom") + contact_pair = (name_geom_1, name_geom_2) + all_contact_pairs.append(contact_pair) + + touch_left_gripper = ( + "red_box", + "vx300s_left/10_left_gripper_finger", + ) in all_contact_pairs + touch_right_gripper = ( + "red_box", + "vx300s_right/10_right_gripper_finger", + ) in all_contact_pairs + touch_table = ("red_box", "table") in all_contact_pairs + + reward = 0 + if touch_right_gripper: + reward = 1 + if touch_right_gripper and not touch_table: # lifted + reward = 2 + if touch_left_gripper: # attempted transfer + reward = 3 + if touch_left_gripper and not touch_table: # successful transfer + reward = 4 + return reward + + +class InsertionEETask(BimanualViperXEETask): + + def __init__(self, random=None): + super().__init__(random=random) + self.max_reward = 4 + + def initialize_episode(self, physics): + """Sets the state of the environment at the start of each episode.""" + self.initialize_robots(physics) + # randomize peg and socket position + peg_pose, socket_pose = sample_insertion_pose() + id2index = (lambda j_id: 16 + (j_id - 16) * 7) # first 16 is robot qpos, 7 is pose dim # hacky + + peg_start_id = physics.model.name2id("red_peg_joint", "joint") + peg_start_idx = id2index(peg_start_id) + np.copyto(physics.data.qpos[peg_start_idx:peg_start_idx + 7], peg_pose) + # print(f"randomized cube position to {cube_position}") + + socket_start_id = physics.model.name2id("blue_socket_joint", "joint") + socket_start_idx = id2index(socket_start_id) + np.copyto(physics.data.qpos[socket_start_idx:socket_start_idx + 7], socket_pose) + # print(f"randomized cube position to {cube_position}") + + super().initialize_episode(physics) + + @staticmethod + def get_env_state(physics): + env_state = physics.data.qpos.copy()[16:] + return env_state + + def get_reward(self, physics): + # return whether peg touches the pin + all_contact_pairs = [] + for i_contact in range(physics.data.ncon): + id_geom_1 = physics.data.contact[i_contact].geom1 + id_geom_2 = physics.data.contact[i_contact].geom2 + name_geom_1 = physics.model.id2name(id_geom_1, "geom") + name_geom_2 = physics.model.id2name(id_geom_2, "geom") + contact_pair = (name_geom_1, name_geom_2) + all_contact_pairs.append(contact_pair) + + touch_right_gripper = ( + "red_peg", + "vx300s_right/10_right_gripper_finger", + ) in all_contact_pairs + touch_left_gripper = (("socket-1", "vx300s_left/10_left_gripper_finger") in all_contact_pairs + or ("socket-2", "vx300s_left/10_left_gripper_finger") in all_contact_pairs + or ("socket-3", "vx300s_left/10_left_gripper_finger") in all_contact_pairs + or ("socket-4", "vx300s_left/10_left_gripper_finger") in all_contact_pairs) + + peg_touch_table = ("red_peg", "table") in all_contact_pairs + socket_touch_table = (("socket-1", "table") in all_contact_pairs or ("socket-2", "table") in all_contact_pairs + or ("socket-3", "table") in all_contact_pairs + or ("socket-4", "table") in all_contact_pairs) + peg_touch_socket = (("red_peg", "socket-1") in all_contact_pairs or ("red_peg", "socket-2") in all_contact_pairs + or ("red_peg", "socket-3") in all_contact_pairs + or ("red_peg", "socket-4") in all_contact_pairs) + pin_touched = ("red_peg", "pin") in all_contact_pairs + + reward = 0 + if touch_left_gripper and touch_right_gripper: # touch both + reward = 1 + if (touch_left_gripper and touch_right_gripper and (not peg_touch_table) + and (not socket_touch_table)): # grasp both + reward = 2 + if (peg_touch_socket and (not peg_touch_table) and (not socket_touch_table)): # peg and socket touching + reward = 3 + if pin_touched: # successful insertion + reward = 4 + return reward diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/eval.sh b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/eval.sh new file mode 100755 index 00000000..2e918d1c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/eval.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# == keep unchanged == +policy_name=ACT +task_name=${1} +task_config=${2} +ckpt_setting=${3} +expert_data_num=${4} +seed=${5} +gpu_id=${6} +# temporal_agg=${5} # use temporal_agg +DEBUG=False + +export CUDA_VISIBLE_DEVICES=${gpu_id} +echo -e "\033[33mgpu id (to use): ${gpu_id}\033[0m" + +cd ../.. + +PYTHONWARNINGS=ignore::UserWarning \ +python script/eval_policy.py --config policy/$policy_name/deploy_policy.yml \ + --overrides \ + --task_name ${task_name} \ + --task_config ${task_config} \ + --ckpt_setting ${ckpt_setting} \ + --ckpt_dir policy/ACT/act_ckpt/act-${task_name}/${ckpt_setting}-${expert_data_num} \ + --seed ${seed} \ + --temporal_agg true \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/imitate_episodes.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/imitate_episodes.py new file mode 100644 index 00000000..15b490b4 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/imitate_episodes.py @@ -0,0 +1,491 @@ +import os + +# Set rendering backend for MuJoCo +os.environ["MUJOCO_GL"] = "egl" + +import torch +import numpy as np +import pickle +import argparse + +import matplotlib + +matplotlib.use("Agg") + +import matplotlib.pyplot as plt +from copy import deepcopy +from tqdm import tqdm +from einops import rearrange + +from constants import DT +from constants import PUPPET_GRIPPER_JOINT_OPEN +from utils import load_data # data functions +from utils import sample_box_pose, sample_insertion_pose # robot functions +from utils import compute_dict_mean, set_seed, detach_dict # helper functions +from act_policy import ACTPolicy, CNNMLPPolicy +from visualize_episodes import save_videos + +from sim_env import BOX_POSE + +import IPython + +e = IPython.embed + + +def main(args): + set_seed(1) + # command line parameters + is_eval = args["eval"] + ckpt_dir = args["ckpt_dir"] + policy_class = args["policy_class"] + onscreen_render = args["onscreen_render"] + task_name = args["task_name"] + batch_size_train = args["batch_size"] + batch_size_val = args["batch_size"] + num_epochs = args["num_epochs"] + + # get task parameters + is_sim = task_name[:4] == "sim-" + if is_sim: + from constants import SIM_TASK_CONFIGS + + task_config = SIM_TASK_CONFIGS[task_name] + else: + from aloha_scripts.constants import TASK_CONFIGS + + task_config = TASK_CONFIGS[task_name] + dataset_dir = task_config["dataset_dir"] + num_episodes = task_config["num_episodes"] + episode_len = task_config["episode_len"] + camera_names = task_config["camera_names"] + + # fixed parameters + state_dim = 14 # yiheng + lr_backbone = 1e-5 + backbone = "resnet18" + if policy_class == "ACT": + enc_layers = 4 + dec_layers = 7 + nheads = 8 + policy_config = { + "lr": args["lr"], + "num_queries": args["chunk_size"], + "kl_weight": args["kl_weight"], + "hidden_dim": args["hidden_dim"], + "dim_feedforward": args["dim_feedforward"], + "lr_backbone": lr_backbone, + "backbone": backbone, + "enc_layers": enc_layers, + "dec_layers": dec_layers, + "nheads": nheads, + "camera_names": camera_names, + } + elif policy_class == "CNNMLP": + policy_config = { + "lr": args["lr"], + "lr_backbone": lr_backbone, + "backbone": backbone, + "num_queries": 1, + "camera_names": camera_names, + } + else: + raise NotImplementedError + + config = { + "num_epochs": num_epochs, + "ckpt_dir": ckpt_dir, + "episode_len": episode_len, + "state_dim": state_dim, + "lr": args["lr"], + "policy_class": policy_class, + "onscreen_render": onscreen_render, + "policy_config": policy_config, + "task_name": task_name, + "seed": args["seed"], + "temporal_agg": args["temporal_agg"], + "camera_names": camera_names, + "real_robot": not is_sim, + "save_freq": args['save_freq'] + } + + if is_eval: + ckpt_names = [f"policy_best.ckpt"] + results = [] + for ckpt_name in ckpt_names: + success_rate, avg_return = eval_bc(config, ckpt_name, save_episode=True) + results.append([ckpt_name, success_rate, avg_return]) + + for ckpt_name, success_rate, avg_return in results: + print(f"{ckpt_name}: {success_rate=} {avg_return=}") + print() + exit() + + train_dataloader, val_dataloader, stats, _ = load_data(dataset_dir, num_episodes, camera_names, batch_size_train, + batch_size_val) + + # save dataset stats + if not os.path.isdir(ckpt_dir): + os.makedirs(ckpt_dir) + stats_path = os.path.join(ckpt_dir, f"dataset_stats.pkl") + with open(stats_path, "wb") as f: + pickle.dump(stats, f) + best_ckpt_info = train_bc(train_dataloader, val_dataloader, config) + best_epoch, min_val_loss, best_state_dict = best_ckpt_info + + # save best checkpoint + ckpt_path = os.path.join(ckpt_dir, f"policy_best.ckpt") + torch.save(best_state_dict, ckpt_path) + print(f"Best ckpt, val loss {min_val_loss:.6f} @ epoch{best_epoch}") + + +def make_policy(policy_class, policy_config): + if policy_class == "ACT": + policy = ACTPolicy(policy_config) + elif policy_class == "CNNMLP": + policy = CNNMLPPolicy(policy_config) + else: + raise NotImplementedError + return policy + + +def make_optimizer(policy_class, policy): + if policy_class == "ACT": + optimizer = policy.configure_optimizers() + elif policy_class == "CNNMLP": + optimizer = policy.configure_optimizers() + else: + raise NotImplementedError + return optimizer + + +def get_image(ts, camera_names): + curr_images = [] + for cam_name in camera_names: + curr_image = rearrange(ts.observation["images"][cam_name], "h w c -> c h w") + curr_images.append(curr_image) + curr_image = np.stack(curr_images, axis=0) + curr_image = torch.from_numpy(curr_image / 255.0).float().cuda().unsqueeze(0) + return curr_image + + +def eval_bc(config, ckpt_name, save_episode=True): + set_seed(1000) + ckpt_dir = config["ckpt_dir"] + state_dim = config["state_dim"] + real_robot = config["real_robot"] + policy_class = config["policy_class"] + onscreen_render = config["onscreen_render"] + policy_config = config["policy_config"] + camera_names = config["camera_names"] + max_timesteps = config["episode_len"] + task_name = config["task_name"] + temporal_agg = config["temporal_agg"] + onscreen_cam = "angle" + + # load policy and stats + ckpt_path = os.path.join(ckpt_dir, ckpt_name) + policy = make_policy(policy_class, policy_config) + loading_status = policy.load_state_dict(torch.load(ckpt_path)) + print(loading_status) + policy.cuda() + policy.eval() + print(f"Loaded: {ckpt_path}") + stats_path = os.path.join(ckpt_dir, f"dataset_stats.pkl") + with open(stats_path, "rb") as f: + stats = pickle.load(f) + + pre_process = lambda s_qpos: (s_qpos - stats["qpos_mean"]) / stats["qpos_std"] + post_process = lambda a: a * stats["action_std"] + stats["action_mean"] + + # load environment + if real_robot: + from aloha_scripts.robot_utils import move_grippers # requires aloha + from aloha_scripts.real_env import make_real_env # requires aloha + + env = make_real_env(init_node=True) + env_max_reward = 0 + else: + from sim_env import make_sim_env + + env = make_sim_env(task_name) + env_max_reward = env.task.max_reward + + query_frequency = policy_config["num_queries"] + if temporal_agg: + query_frequency = 1 + num_queries = policy_config["num_queries"] + + max_timesteps = int(max_timesteps * 1) # may increase for real-world tasks + + num_rollouts = 50 + episode_returns = [] + highest_rewards = [] + for rollout_id in range(num_rollouts): + rollout_id += 0 + ### set task + if "sim_transfer_cube" in task_name: + BOX_POSE[0] = sample_box_pose() # used in sim reset + elif "sim_insertion" in task_name: + BOX_POSE[0] = np.concatenate(sample_insertion_pose()) # used in sim reset + + ts = env.reset() + + ### onscreen render + if onscreen_render: + ax = plt.subplot() + plt_img = ax.imshow(env._physics.render(height=480, width=640, camera_id=onscreen_cam)) + plt.ion() + + ### evaluation loop + if temporal_agg: + all_time_actions = torch.zeros([max_timesteps, max_timesteps + num_queries, state_dim]).cuda() + + qpos_history = torch.zeros((1, max_timesteps, state_dim)).cuda() + image_list = [] # for visualization + qpos_list = [] + target_qpos_list = [] + rewards = [] + with torch.inference_mode(): + for t in range(max_timesteps): + ### update onscreen render and wait for DT + if onscreen_render: + image = env._physics.render(height=480, width=640, camera_id=onscreen_cam) + plt_img.set_data(image) + plt.pause(DT) + + ### process previous timestep to get qpos and image_list + obs = ts.observation + if "images" in obs: + image_list.append(obs["images"]) + else: + image_list.append({"main": obs["image"]}) + qpos_numpy = np.array(obs["qpos"]) + qpos = pre_process(qpos_numpy) + qpos = torch.from_numpy(qpos).float().cuda().unsqueeze(0) + qpos_history[:, t] = qpos + curr_image = get_image(ts, camera_names) + + ### query policy + if config["policy_class"] == "ACT": + if t % query_frequency == 0: + all_actions = policy(qpos, curr_image) + if temporal_agg: + all_time_actions[[t], t:t + num_queries] = all_actions + actions_for_curr_step = all_time_actions[:, t] + actions_populated = torch.all(actions_for_curr_step != 0, axis=1) + actions_for_curr_step = actions_for_curr_step[actions_populated] + k = 0.01 + exp_weights = np.exp(-k * np.arange(len(actions_for_curr_step))) + exp_weights = exp_weights / exp_weights.sum() + exp_weights = (torch.from_numpy(exp_weights).cuda().unsqueeze(dim=1)) + raw_action = (actions_for_curr_step * exp_weights).sum(dim=0, keepdim=True) + else: + raw_action = all_actions[:, t % query_frequency] + elif config["policy_class"] == "CNNMLP": + raw_action = policy(qpos, curr_image) + else: + raise NotImplementedError + + ### post-process actions + raw_action = raw_action.squeeze(0).cpu().numpy() + action = post_process(raw_action) + target_qpos = action + + ### step the environment + ts = env.step(target_qpos) + + ### for visualization + qpos_list.append(qpos_numpy) + target_qpos_list.append(target_qpos) + rewards.append(ts.reward) + + plt.close() + if real_robot: + move_grippers( + [env.puppet_bot_left, env.puppet_bot_right], + [PUPPET_GRIPPER_JOINT_OPEN] * 2, + move_time=0.5, + ) # open + pass + + rewards = np.array(rewards) + episode_return = np.sum(rewards[rewards != None]) + episode_returns.append(episode_return) + episode_highest_reward = np.max(rewards) + highest_rewards.append(episode_highest_reward) + print( + f"Rollout {rollout_id}\n{episode_return=}, {episode_highest_reward=}, {env_max_reward=}, Success: {episode_highest_reward==env_max_reward}" + ) + + if save_episode: + save_videos( + image_list, + DT, + video_path=os.path.join(ckpt_dir, f"video{rollout_id}.mp4"), + ) + + success_rate = np.mean(np.array(highest_rewards) == env_max_reward) + avg_return = np.mean(episode_returns) + summary_str = f"\nSuccess rate: {success_rate}\nAverage return: {avg_return}\n\n" + for r in range(env_max_reward + 1): + more_or_equal_r = (np.array(highest_rewards) >= r).sum() + more_or_equal_r_rate = more_or_equal_r / num_rollouts + summary_str += f"Reward >= {r}: {more_or_equal_r}/{num_rollouts} = {more_or_equal_r_rate*100}%\n" + + # save success rate to txt + result_file_name = "result_" + ckpt_name.split(".")[0] + ".txt" + with open(os.path.join(ckpt_dir, result_file_name), "w") as f: + f.write(summary_str) + f.write(repr(episode_returns)) + f.write("\n\n") + f.write(repr(highest_rewards)) + + return success_rate, avg_return + + +def forward_pass(data, policy): + image_data, qpos_data, action_data, is_pad = data + image_data, qpos_data, action_data, is_pad = ( + image_data.cuda(), + qpos_data.cuda(), + action_data.cuda(), + is_pad.cuda(), + ) + return policy(qpos_data, image_data, action_data, is_pad) # TODO remove None + + +def train_bc(train_dataloader, val_dataloader, config): + num_epochs = config["num_epochs"] + ckpt_dir = config["ckpt_dir"] + seed = config["seed"] + policy_class = config["policy_class"] + policy_config = config["policy_config"] + + set_seed(seed) + + policy = make_policy(policy_class, policy_config) + policy.cuda() + optimizer = make_optimizer(policy_class, policy) + + train_history = [] + validation_history = [] + min_val_loss = np.inf + best_ckpt_info = None + + for epoch in tqdm(range(num_epochs)): + print(f"\nEpoch {epoch}") + # validation + with torch.inference_mode(): + policy.eval() + epoch_dicts = [] + for batch_idx, data in enumerate(val_dataloader): + forward_dict = forward_pass(data, policy) + epoch_dicts.append(forward_dict) + epoch_summary = compute_dict_mean(epoch_dicts) + validation_history.append(epoch_summary) + + epoch_val_loss = epoch_summary["loss"] + if epoch_val_loss < min_val_loss: + min_val_loss = epoch_val_loss + best_ckpt_info = (epoch, min_val_loss, deepcopy(policy.state_dict())) + print(f"Val loss: {epoch_val_loss:.5f}") + summary_string = "" + for k, v in epoch_summary.items(): + summary_string += f"{k}: {v.item():.3f} " + + # training + policy.train() + optimizer.zero_grad() + for batch_idx, data in enumerate(train_dataloader): + forward_dict = forward_pass(data, policy) + # backward + loss = forward_dict["loss"] + loss.backward() + optimizer.step() + optimizer.zero_grad() + train_history.append(detach_dict(forward_dict)) + epoch_summary = compute_dict_mean(train_history[(batch_idx + 1) * epoch:(batch_idx + 1) * (epoch + 1)]) + epoch_train_loss = epoch_summary["loss"] + print(f"Train loss: {epoch_train_loss:.5f}") + summary_string = "" + for k, v in epoch_summary.items(): + summary_string += f"{k}: {v.item():.3f} " + + if (epoch + 1) % config['save_freq'] == 0: + ckpt_path = os.path.join(ckpt_dir, f"policy_epoch_{epoch + 1}_seed_{seed}.ckpt") + torch.save(policy.state_dict(), ckpt_path) + plot_history(train_history, validation_history, epoch, ckpt_dir, seed) + + ckpt_path = os.path.join(ckpt_dir, f"policy_last.ckpt") + torch.save(policy.state_dict(), ckpt_path) + + best_epoch, min_val_loss, best_state_dict = best_ckpt_info + ckpt_path = os.path.join(ckpt_dir, f"policy_epoch_{best_epoch}_seed_{seed}.ckpt") + torch.save(best_state_dict, ckpt_path) + print(f"Training finished:\nSeed {seed}, val loss {min_val_loss:.6f} at epoch {best_epoch}") + + # save training curves + plot_history(train_history, validation_history, num_epochs, ckpt_dir, seed) + + return best_ckpt_info + + +def plot_history(train_history, validation_history, num_epochs, ckpt_dir, seed): + # save training curves + for key in train_history[0]: + plot_path = os.path.join(ckpt_dir, f"train_val_{key}_seed_{seed}.png") + plt.figure() + train_values = [summary[key].item() for summary in train_history] + val_values = [summary[key].item() for summary in validation_history] + plt.plot( + np.linspace(0, num_epochs - 1, len(train_history)), + train_values, + label="train", + ) + plt.plot( + np.linspace(0, num_epochs - 1, len(validation_history)), + val_values, + label="validation", + ) + # plt.ylim([-0.1, 1]) + plt.tight_layout() + plt.legend() + plt.title(key) + plt.savefig(plot_path) + print(f"Saved plots to {ckpt_dir}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--eval", action="store_true") + parser.add_argument("--onscreen_render", action="store_true") + parser.add_argument("--ckpt_dir", action="store", type=str, help="ckpt_dir", required=True) + parser.add_argument( + "--policy_class", + action="store", + type=str, + help="policy_class, capitalize", + required=True, + ) + parser.add_argument("--task_name", action="store", type=str, help="task_name", required=True) + parser.add_argument("--batch_size", action="store", type=int, help="batch_size", required=True) + parser.add_argument("--seed", action="store", type=int, help="seed", required=True) + parser.add_argument("--num_epochs", action="store", type=int, help="num_epochs", required=True) + parser.add_argument("--lr", action="store", type=float, help="lr", required=True) + + # for ACT + parser.add_argument("--kl_weight", action="store", type=int, help="KL Weight", required=False) + parser.add_argument("--chunk_size", action="store", type=int, help="chunk_size", required=False) + parser.add_argument("--hidden_dim", action="store", type=int, help="hidden_dim", required=False) + parser.add_argument("--state_dim", action="store", type=int, help="state dim", required=True) + parser.add_argument("--save_freq", action="store", type=int, help="save ckpt frequency", required=False, default=6000) + parser.add_argument( + "--dim_feedforward", + action="store", + type=int, + help="dim_feedforward", + required=False, + ) + parser.add_argument("--temporal_agg", action="store_true") + + main(vars(parser.parse_args())) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/process_data.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/process_data.py new file mode 100644 index 00000000..09ce07a7 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/process_data.py @@ -0,0 +1,168 @@ +import sys + +sys.path.append("./policy/ACT/") + +import os +import h5py +import numpy as np +import pickle +import cv2 +import argparse +import pdb +import json + + +def load_hdf5(dataset_path): + if not os.path.isfile(dataset_path): + print(f"Dataset does not exist at \n{dataset_path}\n") + exit() + + with h5py.File(dataset_path, "r") as root: + left_gripper, left_arm = ( + root["/joint_action/left_gripper"][()], + root["/joint_action/left_arm"][()], + ) + right_gripper, right_arm = ( + root["/joint_action/right_gripper"][()], + root["/joint_action/right_arm"][()], + ) + image_dict = dict() + for cam_name in root[f"/observation/"].keys(): + image_dict[cam_name] = root[f"/observation/{cam_name}/rgb"][()] + + return left_gripper, left_arm, right_gripper, right_arm, image_dict + + +def images_encoding(imgs): + encode_data = [] + padded_data = [] + max_len = 0 + for i in range(len(imgs)): + success, encoded_image = cv2.imencode(".jpg", imgs[i]) + jpeg_data = encoded_image.tobytes() + encode_data.append(jpeg_data) + max_len = max(max_len, len(jpeg_data)) + # padding + for i in range(len(imgs)): + padded_data.append(encode_data[i].ljust(max_len, b"\0")) + return encode_data, max_len + + +def data_transform(path, episode_num, save_path): + begin = 0 + floders = os.listdir(path) + assert episode_num <= len(floders), "data num not enough" + + if not os.path.exists(save_path): + os.makedirs(save_path) + + for i in range(episode_num): + left_gripper_all, left_arm_all, right_gripper_all, right_arm_all, image_dict = (load_hdf5( + os.path.join(path, f"episode{i}.hdf5"))) + qpos = [] + actions = [] + cam_high = [] + cam_right_wrist = [] + cam_left_wrist = [] + left_arm_dim = [] + right_arm_dim = [] + + last_state = None + for j in range(0, left_gripper_all.shape[0]): + + left_gripper, left_arm, right_gripper, right_arm = ( + left_gripper_all[j], + left_arm_all[j], + right_gripper_all[j], + right_arm_all[j], + ) + + if j != left_gripper_all.shape[0] - 1: + state = np.concatenate((left_arm, [left_gripper], right_arm, [right_gripper]), axis=0) # joint + + state = state.astype(np.float32) + qpos.append(state) + + camera_high_bits = image_dict["head_camera"][j] + camera_high = cv2.imdecode(np.frombuffer(camera_high_bits, np.uint8), cv2.IMREAD_COLOR) + camera_high_resized = cv2.resize(camera_high, (640, 480)) + cam_high.append(camera_high_resized) + + camera_right_wrist_bits = image_dict["right_camera"][j] + camera_right_wrist = cv2.imdecode(np.frombuffer(camera_right_wrist_bits, np.uint8), cv2.IMREAD_COLOR) + camera_right_wrist_resized = cv2.resize(camera_right_wrist, (640, 480)) + cam_right_wrist.append(camera_right_wrist_resized) + + camera_left_wrist_bits = image_dict["left_camera"][j] + camera_left_wrist = cv2.imdecode(np.frombuffer(camera_left_wrist_bits, np.uint8), cv2.IMREAD_COLOR) + camera_left_wrist_resized = cv2.resize(camera_left_wrist, (640, 480)) + cam_left_wrist.append(camera_left_wrist_resized) + + if j != 0: + action = state + actions.append(action) + left_arm_dim.append(left_arm.shape[0]) + right_arm_dim.append(right_arm.shape[0]) + + hdf5path = os.path.join(save_path, f"episode_{i}.hdf5") + + with h5py.File(hdf5path, "w") as f: + f.create_dataset("action", data=np.array(actions)) + obs = f.create_group("observations") + obs.create_dataset("qpos", data=np.array(qpos)) + obs.create_dataset("left_arm_dim", data=np.array(left_arm_dim)) + obs.create_dataset("right_arm_dim", data=np.array(right_arm_dim)) + image = obs.create_group("images") + # cam_high_enc, len_high = images_encoding(cam_high) + # cam_right_wrist_enc, len_right = images_encoding(cam_right_wrist) + # cam_left_wrist_enc, len_left = images_encoding(cam_left_wrist) + image.create_dataset("cam_high", data=np.stack(cam_high), dtype=np.uint8) + image.create_dataset("cam_right_wrist", data=np.stack(cam_right_wrist), dtype=np.uint8) + image.create_dataset("cam_left_wrist", data=np.stack(cam_left_wrist), dtype=np.uint8) + + begin += 1 + print(f"proccess {i} success!") + + return begin + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Process some episodes.") + parser.add_argument( + "task_name", + type=str, + help="The name of the task (e.g., adjust_bottle)", + ) + parser.add_argument("task_config", type=str) + parser.add_argument("expert_data_num", type=int) + + args = parser.parse_args() + + task_name = args.task_name + task_config = args.task_config + expert_data_num = args.expert_data_num + + begin = 0 + begin = data_transform( + os.path.join("../../data/", task_name, task_config, 'data'), + expert_data_num, + f"processed_data/sim-{task_name}/{task_config}-{expert_data_num}", + ) + + SIM_TASK_CONFIGS_PATH = "./SIM_TASK_CONFIGS.json" + + try: + with open(SIM_TASK_CONFIGS_PATH, "r") as f: + SIM_TASK_CONFIGS = json.load(f) + except Exception: + SIM_TASK_CONFIGS = {} + + SIM_TASK_CONFIGS[f"sim-{task_name}-{task_config}-{expert_data_num}"] = { + "dataset_dir": f"./processed_data/sim-{task_name}/{task_config}-{expert_data_num}", + "num_episodes": expert_data_num, + "episode_len": 1000, + "camera_names": ["cam_high", "cam_right_wrist", "cam_left_wrist"], + } + + with open(SIM_TASK_CONFIGS_PATH, "w") as f: + json.dump(SIM_TASK_CONFIGS, f, indent=4) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/process_data.sh b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/process_data.sh new file mode 100644 index 00000000..cff4a950 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/process_data.sh @@ -0,0 +1,5 @@ +task_name=${1} +task_config=${2} +expert_data_num=${3} + +python process_data.py $task_name $task_config $expert_data_num \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/record_sim_episodes.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/record_sim_episodes.py new file mode 100644 index 00000000..9ddcebc9 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/record_sim_episodes.py @@ -0,0 +1,201 @@ +import time +import os +import numpy as np +import argparse +import matplotlib.pyplot as plt +import h5py + +from constants import PUPPET_GRIPPER_POSITION_NORMALIZE_FN, SIM_TASK_CONFIGS +from ee_sim_env import make_ee_sim_env +from sim_env import make_sim_env, BOX_POSE +from scripted_policy import PickAndTransferPolicy, InsertionPolicy + +import IPython + +e = IPython.embed + + +def main(args): + """ + Generate demonstration data in simulation. + First rollout the policy (defined in ee space) in ee_sim_env. Obtain the joint trajectory. + Replace the gripper joint positions with the commanded joint position. + Replay this joint trajectory (as action sequence) in sim_env, and record all observations. + Save this episode of data, and continue to next episode of data collection. + """ + + task_name = args["task_name"] + dataset_dir = args["dataset_dir"] + num_episodes = args["num_episodes"] + onscreen_render = args["onscreen_render"] + inject_noise = False + render_cam_name = "angle" + + if not os.path.isdir(dataset_dir): + os.makedirs(dataset_dir, exist_ok=True) + + episode_len = SIM_TASK_CONFIGS[task_name]["episode_len"] + camera_names = SIM_TASK_CONFIGS[task_name]["camera_names"] + if task_name == "sim_transfer_cube_scripted": + policy_cls = PickAndTransferPolicy + elif task_name == "sim_insertion_scripted": + policy_cls = InsertionPolicy + else: + raise NotImplementedError + + success = [] + for episode_idx in range(num_episodes): + print(f"{episode_idx=}") + print("Rollout out EE space scripted policy") + # setup the environment + env = make_ee_sim_env(task_name) + ts = env.reset() + episode = [ts] + policy = policy_cls(inject_noise) + # setup plotting + if onscreen_render: + ax = plt.subplot() + plt_img = ax.imshow(ts.observation["images"][render_cam_name]) + plt.ion() + for step in range(episode_len): + action = policy(ts) + ts = env.step(action) + episode.append(ts) + if onscreen_render: + plt_img.set_data(ts.observation["images"][render_cam_name]) + plt.pause(0.002) + plt.close() + + episode_return = np.sum([ts.reward for ts in episode[1:]]) + episode_max_reward = np.max([ts.reward for ts in episode[1:]]) + if episode_max_reward == env.task.max_reward: + print(f"{episode_idx=} Successful, {episode_return=}") + else: + print(f"{episode_idx=} Failed") + + joint_traj = [ts.observation["qpos"] for ts in episode] + # replace gripper pose with gripper control + gripper_ctrl_traj = [ts.observation["gripper_ctrl"] for ts in episode] + for joint, ctrl in zip(joint_traj, gripper_ctrl_traj): + left_ctrl = PUPPET_GRIPPER_POSITION_NORMALIZE_FN(ctrl[0]) + right_ctrl = PUPPET_GRIPPER_POSITION_NORMALIZE_FN(ctrl[2]) + joint[6] = left_ctrl + joint[6 + 7] = right_ctrl + + subtask_info = episode[0].observation["env_state"].copy() # box pose at step 0 + + # clear unused variables + del env + del episode + del policy + + # setup the environment + print("Replaying joint commands") + env = make_sim_env(task_name) + BOX_POSE[0] = ( + subtask_info # make sure the sim_env has the same object configurations as ee_sim_env + ) + ts = env.reset() + + episode_replay = [ts] + # setup plotting + if onscreen_render: + ax = plt.subplot() + plt_img = ax.imshow(ts.observation["images"][render_cam_name]) + plt.ion() + for t in range(len(joint_traj)): # note: this will increase episode length by 1 + action = joint_traj[t] + ts = env.step(action) + episode_replay.append(ts) + if onscreen_render: + plt_img.set_data(ts.observation["images"][render_cam_name]) + plt.pause(0.02) + + episode_return = np.sum([ts.reward for ts in episode_replay[1:]]) + episode_max_reward = np.max([ts.reward for ts in episode_replay[1:]]) + if episode_max_reward == env.task.max_reward: + success.append(1) + print(f"{episode_idx=} Successful, {episode_return=}") + else: + success.append(0) + print(f"{episode_idx=} Failed") + + plt.close() + """ + For each timestep: + observations + - images + - each_cam_name (480, 640, 3) 'uint8' + - qpos (14,) 'float64' + - qvel (14,) 'float64' + + action (14,) 'float64' + """ + + data_dict = { + "/observations/qpos": [], + "/observations/qvel": [], + "/action": [], + } + for cam_name in camera_names: + data_dict[f"/observations/images/{cam_name}"] = [] + + # because the replaying, there will be eps_len + 1 actions and eps_len + 2 timesteps + # truncate here to be consistent + joint_traj = joint_traj[:-1] + episode_replay = episode_replay[:-1] + + # len(joint_traj) i.e. actions: max_timesteps + # len(episode_replay) i.e. time steps: max_timesteps + 1 + max_timesteps = len(joint_traj) + while joint_traj: + action = joint_traj.pop(0) + ts = episode_replay.pop(0) + data_dict["/observations/qpos"].append(ts.observation["qpos"]) + data_dict["/observations/qvel"].append(ts.observation["qvel"]) + data_dict["/action"].append(action) + for cam_name in camera_names: + data_dict[f"/observations/images/{cam_name}"].append(ts.observation["images"][cam_name]) + + # HDF5 + t0 = time.time() + dataset_path = os.path.join(dataset_dir, f"episode_{episode_idx}") + with h5py.File(dataset_path + ".hdf5", "w", rdcc_nbytes=1024**2 * 2) as root: + root.attrs["sim"] = True + obs = root.create_group("observations") + image = obs.create_group("images") + for cam_name in camera_names: + _ = image.create_dataset( + cam_name, + (max_timesteps, 480, 640, 3), + dtype="uint8", + chunks=(1, 480, 640, 3), + ) + # compression='gzip',compression_opts=2,) + # compression=32001, compression_opts=(0, 0, 0, 0, 9, 1, 1), shuffle=False) + qpos = obs.create_dataset("qpos", (max_timesteps, 14)) + qvel = obs.create_dataset("qvel", (max_timesteps, 14)) + action = root.create_dataset("action", (max_timesteps, 14)) + + for name, array in data_dict.items(): + root[name][...] = array + print(f"Saving: {time.time() - t0:.1f} secs\n") + + print(f"Saved to {dataset_dir}") + print(f"Success: {np.sum(success)} / {len(success)}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--task_name", action="store", type=str, help="task_name", required=True) + parser.add_argument( + "--dataset_dir", + action="store", + type=str, + help="dataset saving dir", + required=True, + ) + parser.add_argument("--num_episodes", action="store", type=int, help="num_episodes", required=False) + parser.add_argument("--onscreen_render", action="store_true") + + main(vars(parser.parse_args())) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/scripted_policy.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/scripted_policy.py new file mode 100644 index 00000000..1f7beae9 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/scripted_policy.py @@ -0,0 +1,341 @@ +import numpy as np +import matplotlib.pyplot as plt +from pyquaternion import Quaternion + +from constants import SIM_TASK_CONFIGS +from ee_sim_env import make_ee_sim_env + +import IPython + +e = IPython.embed + + +class BasePolicy: + + def __init__(self, inject_noise=False): + self.inject_noise = inject_noise + self.step_count = 0 + self.left_trajectory = None + self.right_trajectory = None + + def generate_trajectory(self, ts_first): + raise NotImplementedError + + @staticmethod + def interpolate(curr_waypoint, next_waypoint, t): + t_frac = (t - curr_waypoint["t"]) / (next_waypoint["t"] - curr_waypoint["t"]) + curr_xyz = curr_waypoint["xyz"] + curr_quat = curr_waypoint["quat"] + curr_grip = curr_waypoint["gripper"] + next_xyz = next_waypoint["xyz"] + next_quat = next_waypoint["quat"] + next_grip = next_waypoint["gripper"] + xyz = curr_xyz + (next_xyz - curr_xyz) * t_frac + quat = curr_quat + (next_quat - curr_quat) * t_frac + gripper = curr_grip + (next_grip - curr_grip) * t_frac + return xyz, quat, gripper + + def __call__(self, ts): + # generate trajectory at first timestep, then open-loop execution + if self.step_count == 0: + self.generate_trajectory(ts) + + # obtain left and right waypoints + if self.left_trajectory[0]["t"] == self.step_count: + self.curr_left_waypoint = self.left_trajectory.pop(0) + next_left_waypoint = self.left_trajectory[0] + + if self.right_trajectory[0]["t"] == self.step_count: + self.curr_right_waypoint = self.right_trajectory.pop(0) + next_right_waypoint = self.right_trajectory[0] + + # interpolate between waypoints to obtain current pose and gripper command + left_xyz, left_quat, left_gripper = self.interpolate(self.curr_left_waypoint, next_left_waypoint, + self.step_count) + right_xyz, right_quat, right_gripper = self.interpolate(self.curr_right_waypoint, next_right_waypoint, + self.step_count) + + # Inject noise + if self.inject_noise: + scale = 0.01 + left_xyz = left_xyz + np.random.uniform(-scale, scale, left_xyz.shape) + right_xyz = right_xyz + np.random.uniform(-scale, scale, right_xyz.shape) + + action_left = np.concatenate([left_xyz, left_quat, [left_gripper]]) + action_right = np.concatenate([right_xyz, right_quat, [right_gripper]]) + + self.step_count += 1 + return np.concatenate([action_left, action_right]) + + +class PickAndTransferPolicy(BasePolicy): + + def generate_trajectory(self, ts_first): + init_mocap_pose_right = ts_first.observation["mocap_pose_right"] + init_mocap_pose_left = ts_first.observation["mocap_pose_left"] + + box_info = np.array(ts_first.observation["env_state"]) + box_xyz = box_info[:3] + box_quat = box_info[3:] + # print(f"Generate trajectory for {box_xyz=}") + + gripper_pick_quat = Quaternion(init_mocap_pose_right[3:]) + gripper_pick_quat = gripper_pick_quat * Quaternion(axis=[0.0, 1.0, 0.0], degrees=-60) + + meet_left_quat = Quaternion(axis=[1.0, 0.0, 0.0], degrees=90) + + meet_xyz = np.array([0, 0.5, 0.25]) + + self.left_trajectory = [ + { + "t": 0, + "xyz": init_mocap_pose_left[:3], + "quat": init_mocap_pose_left[3:], + "gripper": 0, + }, # sleep + { + "t": 100, + "xyz": meet_xyz + np.array([-0.1, 0, -0.02]), + "quat": meet_left_quat.elements, + "gripper": 1, + }, # approach meet position + { + "t": 260, + "xyz": meet_xyz + np.array([0.02, 0, -0.02]), + "quat": meet_left_quat.elements, + "gripper": 1, + }, # move to meet position + { + "t": 310, + "xyz": meet_xyz + np.array([0.02, 0, -0.02]), + "quat": meet_left_quat.elements, + "gripper": 0, + }, # close gripper + { + "t": 360, + "xyz": meet_xyz + np.array([-0.1, 0, -0.02]), + "quat": np.array([1, 0, 0, 0]), + "gripper": 0, + }, # move left + { + "t": 400, + "xyz": meet_xyz + np.array([-0.1, 0, -0.02]), + "quat": np.array([1, 0, 0, 0]), + "gripper": 0, + }, # stay + ] + + self.right_trajectory = [ + { + "t": 0, + "xyz": init_mocap_pose_right[:3], + "quat": init_mocap_pose_right[3:], + "gripper": 0, + }, # sleep + { + "t": 90, + "xyz": box_xyz + np.array([0, 0, 0.08]), + "quat": gripper_pick_quat.elements, + "gripper": 1, + }, # approach the cube + { + "t": 130, + "xyz": box_xyz + np.array([0, 0, -0.015]), + "quat": gripper_pick_quat.elements, + "gripper": 1, + }, # go down + { + "t": 170, + "xyz": box_xyz + np.array([0, 0, -0.015]), + "quat": gripper_pick_quat.elements, + "gripper": 0, + }, # close gripper + { + "t": 200, + "xyz": meet_xyz + np.array([0.05, 0, 0]), + "quat": gripper_pick_quat.elements, + "gripper": 0, + }, # approach meet position + { + "t": 220, + "xyz": meet_xyz, + "quat": gripper_pick_quat.elements, + "gripper": 0, + }, # move to meet position + { + "t": 310, + "xyz": meet_xyz, + "quat": gripper_pick_quat.elements, + "gripper": 1, + }, # open gripper + { + "t": 360, + "xyz": meet_xyz + np.array([0.1, 0, 0]), + "quat": gripper_pick_quat.elements, + "gripper": 1, + }, # move to right + { + "t": 400, + "xyz": meet_xyz + np.array([0.1, 0, 0]), + "quat": gripper_pick_quat.elements, + "gripper": 1, + }, # stay + ] + + +class InsertionPolicy(BasePolicy): + + def generate_trajectory(self, ts_first): + init_mocap_pose_right = ts_first.observation["mocap_pose_right"] + init_mocap_pose_left = ts_first.observation["mocap_pose_left"] + + peg_info = np.array(ts_first.observation["env_state"])[:7] + peg_xyz = peg_info[:3] + peg_quat = peg_info[3:] + + socket_info = np.array(ts_first.observation["env_state"])[7:] + socket_xyz = socket_info[:3] + socket_quat = socket_info[3:] + + gripper_pick_quat_right = Quaternion(init_mocap_pose_right[3:]) + gripper_pick_quat_right = gripper_pick_quat_right * Quaternion(axis=[0.0, 1.0, 0.0], degrees=-60) + + gripper_pick_quat_left = Quaternion(init_mocap_pose_right[3:]) + gripper_pick_quat_left = gripper_pick_quat_left * Quaternion(axis=[0.0, 1.0, 0.0], degrees=60) + + meet_xyz = np.array([0, 0.5, 0.15]) + lift_right = 0.00715 + + self.left_trajectory = [ + { + "t": 0, + "xyz": init_mocap_pose_left[:3], + "quat": init_mocap_pose_left[3:], + "gripper": 0, + }, # sleep + { + "t": 120, + "xyz": socket_xyz + np.array([0, 0, 0.08]), + "quat": gripper_pick_quat_left.elements, + "gripper": 1, + }, # approach the cube + { + "t": 170, + "xyz": socket_xyz + np.array([0, 0, -0.03]), + "quat": gripper_pick_quat_left.elements, + "gripper": 1, + }, # go down + { + "t": 220, + "xyz": socket_xyz + np.array([0, 0, -0.03]), + "quat": gripper_pick_quat_left.elements, + "gripper": 0, + }, # close gripper + { + "t": 285, + "xyz": meet_xyz + np.array([-0.1, 0, 0]), + "quat": gripper_pick_quat_left.elements, + "gripper": 0, + }, # approach meet position + { + "t": 340, + "xyz": meet_xyz + np.array([-0.05, 0, 0]), + "quat": gripper_pick_quat_left.elements, + "gripper": 0, + }, # insertion + { + "t": 400, + "xyz": meet_xyz + np.array([-0.05, 0, 0]), + "quat": gripper_pick_quat_left.elements, + "gripper": 0, + }, # insertion + ] + + self.right_trajectory = [ + { + "t": 0, + "xyz": init_mocap_pose_right[:3], + "quat": init_mocap_pose_right[3:], + "gripper": 0, + }, # sleep + { + "t": 120, + "xyz": peg_xyz + np.array([0, 0, 0.08]), + "quat": gripper_pick_quat_right.elements, + "gripper": 1, + }, # approach the cube + { + "t": 170, + "xyz": peg_xyz + np.array([0, 0, -0.03]), + "quat": gripper_pick_quat_right.elements, + "gripper": 1, + }, # go down + { + "t": 220, + "xyz": peg_xyz + np.array([0, 0, -0.03]), + "quat": gripper_pick_quat_right.elements, + "gripper": 0, + }, # close gripper + { + "t": 285, + "xyz": meet_xyz + np.array([0.1, 0, lift_right]), + "quat": gripper_pick_quat_right.elements, + "gripper": 0, + }, # approach meet position + { + "t": 340, + "xyz": meet_xyz + np.array([0.05, 0, lift_right]), + "quat": gripper_pick_quat_right.elements, + "gripper": 0, + }, # insertion + { + "t": 400, + "xyz": meet_xyz + np.array([0.05, 0, lift_right]), + "quat": gripper_pick_quat_right.elements, + "gripper": 0, + }, # insertion + ] + + +def test_policy(task_name): + # example rolling out pick_and_transfer policy + onscreen_render = True + inject_noise = False + + # setup the environment + episode_len = SIM_TASK_CONFIGS[task_name]["episode_len"] + if "sim_transfer_cube" in task_name: + env = make_ee_sim_env("sim_transfer_cube") + elif "sim_insertion" in task_name: + env = make_ee_sim_env("sim_insertion") + else: + raise NotImplementedError + + for episode_idx in range(2): + ts = env.reset() + episode = [ts] + if onscreen_render: + ax = plt.subplot() + plt_img = ax.imshow(ts.observation["images"]["angle"]) + plt.ion() + + policy = PickAndTransferPolicy(inject_noise) + for step in range(episode_len): + action = policy(ts) + ts = env.step(action) + episode.append(ts) + if onscreen_render: + plt_img.set_data(ts.observation["images"]["angle"]) + plt.pause(0.02) + plt.close() + + episode_return = np.sum([ts.reward for ts in episode[1:]]) + if episode_return > 0: + print(f"{episode_idx=} Successful, {episode_return=}") + else: + print(f"{episode_idx=} Failed") + + +if __name__ == "__main__": + test_task_name = "sim_transfer_cube_scripted" + test_policy(test_task_name) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/sim_env.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/sim_env.py new file mode 100644 index 00000000..a79e718b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/sim_env.py @@ -0,0 +1,319 @@ +import numpy as np +import os +import collections +import matplotlib.pyplot as plt +from dm_control import mujoco +from dm_control.rl import control +from dm_control.suite import base + +from constants import DT, XML_DIR, START_ARM_POSE +from constants import PUPPET_GRIPPER_POSITION_UNNORMALIZE_FN +from constants import MASTER_GRIPPER_POSITION_NORMALIZE_FN +from constants import PUPPET_GRIPPER_POSITION_NORMALIZE_FN +from constants import PUPPET_GRIPPER_VELOCITY_NORMALIZE_FN + +import IPython + +e = IPython.embed + +BOX_POSE = [None] # to be changed from outside + + +def make_sim_env(task_name): + """ + Environment for simulated robot bi-manual manipulation, with joint position control + Action space: [left_arm_qpos (6), # absolute joint position + left_gripper_positions (1), # normalized gripper position (0: close, 1: open) + right_arm_qpos (6), # absolute joint position + right_gripper_positions (1),] # normalized gripper position (0: close, 1: open) + + Observation space: {"qpos": Concat[ left_arm_qpos (6), # absolute joint position + left_gripper_position (1), # normalized gripper position (0: close, 1: open) + right_arm_qpos (6), # absolute joint position + right_gripper_qpos (1)] # normalized gripper position (0: close, 1: open) + "qvel": Concat[ left_arm_qvel (6), # absolute joint velocity (rad) + left_gripper_velocity (1), # normalized gripper velocity (pos: opening, neg: closing) + right_arm_qvel (6), # absolute joint velocity (rad) + right_gripper_qvel (1)] # normalized gripper velocity (pos: opening, neg: closing) + "images": {"main": (480x640x3)} # h, w, c, dtype='uint8' + """ + if "sim_transfer_cube" in task_name: + xml_path = os.path.join(XML_DIR, f"bimanual_viperx_transfer_cube.xml") + physics = mujoco.Physics.from_xml_path(xml_path) + task = TransferCubeTask(random=False) + env = control.Environment( + physics, + task, + time_limit=20, + control_timestep=DT, + n_sub_steps=None, + flat_observation=False, + ) + elif "sim_insertion" in task_name: + xml_path = os.path.join(XML_DIR, f"bimanual_viperx_insertion.xml") + physics = mujoco.Physics.from_xml_path(xml_path) + task = InsertionTask(random=False) + env = control.Environment( + physics, + task, + time_limit=20, + control_timestep=DT, + n_sub_steps=None, + flat_observation=False, + ) + else: + raise NotImplementedError + return env + + +class BimanualViperXTask(base.Task): + + def __init__(self, random=None): + super().__init__(random=random) + + def before_step(self, action, physics): + left_arm_action = action[:6] + right_arm_action = action[7:7 + 6] + normalized_left_gripper_action = action[6] + normalized_right_gripper_action = action[7 + 6] + + left_gripper_action = PUPPET_GRIPPER_POSITION_UNNORMALIZE_FN(normalized_left_gripper_action) + right_gripper_action = PUPPET_GRIPPER_POSITION_UNNORMALIZE_FN(normalized_right_gripper_action) + + full_left_gripper_action = [left_gripper_action, -left_gripper_action] + full_right_gripper_action = [right_gripper_action, -right_gripper_action] + + env_action = np.concatenate([ + left_arm_action, + full_left_gripper_action, + right_arm_action, + full_right_gripper_action, + ]) + super().before_step(env_action, physics) + return + + def initialize_episode(self, physics): + """Sets the state of the environment at the start of each episode.""" + super().initialize_episode(physics) + + @staticmethod + def get_qpos(physics): + qpos_raw = physics.data.qpos.copy() + left_qpos_raw = qpos_raw[:8] + right_qpos_raw = qpos_raw[8:16] + left_arm_qpos = left_qpos_raw[:6] + right_arm_qpos = right_qpos_raw[:6] + left_gripper_qpos = [PUPPET_GRIPPER_POSITION_NORMALIZE_FN(left_qpos_raw[6])] + right_gripper_qpos = [PUPPET_GRIPPER_POSITION_NORMALIZE_FN(right_qpos_raw[6])] + return np.concatenate([left_arm_qpos, left_gripper_qpos, right_arm_qpos, right_gripper_qpos]) + + @staticmethod + def get_qvel(physics): + qvel_raw = physics.data.qvel.copy() + left_qvel_raw = qvel_raw[:8] + right_qvel_raw = qvel_raw[8:16] + left_arm_qvel = left_qvel_raw[:6] + right_arm_qvel = right_qvel_raw[:6] + left_gripper_qvel = [PUPPET_GRIPPER_VELOCITY_NORMALIZE_FN(left_qvel_raw[6])] + right_gripper_qvel = [PUPPET_GRIPPER_VELOCITY_NORMALIZE_FN(right_qvel_raw[6])] + return np.concatenate([left_arm_qvel, left_gripper_qvel, right_arm_qvel, right_gripper_qvel]) + + @staticmethod + def get_env_state(physics): + raise NotImplementedError + + def get_observation(self, physics): + obs = collections.OrderedDict() + obs["qpos"] = self.get_qpos(physics) + obs["qvel"] = self.get_qvel(physics) + obs["env_state"] = self.get_env_state(physics) + obs["images"] = dict() + obs["images"]["top"] = physics.render(height=480, width=640, camera_id="top") + obs["images"]["angle"] = physics.render(height=480, width=640, camera_id="angle") + obs["images"]["vis"] = physics.render(height=480, width=640, camera_id="front_close") + + return obs + + def get_reward(self, physics): + # return whether left gripper is holding the box + raise NotImplementedError + + +class TransferCubeTask(BimanualViperXTask): + + def __init__(self, random=None): + super().__init__(random=random) + self.max_reward = 4 + + def initialize_episode(self, physics): + """Sets the state of the environment at the start of each episode.""" + # TODO Notice: this function does not randomize the env configuration. Instead, set BOX_POSE from outside + # reset qpos, control and box position + with physics.reset_context(): + physics.named.data.qpos[:16] = START_ARM_POSE + np.copyto(physics.data.ctrl, START_ARM_POSE) + assert BOX_POSE[0] is not None + physics.named.data.qpos[-7:] = BOX_POSE[0] + # print(f"{BOX_POSE=}") + super().initialize_episode(physics) + + @staticmethod + def get_env_state(physics): + env_state = physics.data.qpos.copy()[16:] + return env_state + + def get_reward(self, physics): + # return whether left gripper is holding the box + all_contact_pairs = [] + for i_contact in range(physics.data.ncon): + id_geom_1 = physics.data.contact[i_contact].geom1 + id_geom_2 = physics.data.contact[i_contact].geom2 + name_geom_1 = physics.model.id2name(id_geom_1, "geom") + name_geom_2 = physics.model.id2name(id_geom_2, "geom") + contact_pair = (name_geom_1, name_geom_2) + all_contact_pairs.append(contact_pair) + + touch_left_gripper = ( + "red_box", + "vx300s_left/10_left_gripper_finger", + ) in all_contact_pairs + touch_right_gripper = ( + "red_box", + "vx300s_right/10_right_gripper_finger", + ) in all_contact_pairs + touch_table = ("red_box", "table") in all_contact_pairs + + reward = 0 + if touch_right_gripper: + reward = 1 + if touch_right_gripper and not touch_table: # lifted + reward = 2 + if touch_left_gripper: # attempted transfer + reward = 3 + if touch_left_gripper and not touch_table: # successful transfer + reward = 4 + return reward + + +class InsertionTask(BimanualViperXTask): + + def __init__(self, random=None): + super().__init__(random=random) + self.max_reward = 4 + + def initialize_episode(self, physics): + """Sets the state of the environment at the start of each episode.""" + # TODO Notice: this function does not randomize the env configuration. Instead, set BOX_POSE from outside + # reset qpos, control and box position + with physics.reset_context(): + physics.named.data.qpos[:16] = START_ARM_POSE + np.copyto(physics.data.ctrl, START_ARM_POSE) + assert BOX_POSE[0] is not None + physics.named.data.qpos[-7 * 2:] = BOX_POSE[0] # two objects + # print(f"{BOX_POSE=}") + super().initialize_episode(physics) + + @staticmethod + def get_env_state(physics): + env_state = physics.data.qpos.copy()[16:] + return env_state + + def get_reward(self, physics): + # return whether peg touches the pin + all_contact_pairs = [] + for i_contact in range(physics.data.ncon): + id_geom_1 = physics.data.contact[i_contact].geom1 + id_geom_2 = physics.data.contact[i_contact].geom2 + name_geom_1 = physics.model.id2name(id_geom_1, "geom") + name_geom_2 = physics.model.id2name(id_geom_2, "geom") + contact_pair = (name_geom_1, name_geom_2) + all_contact_pairs.append(contact_pair) + + touch_right_gripper = ( + "red_peg", + "vx300s_right/10_right_gripper_finger", + ) in all_contact_pairs + touch_left_gripper = (("socket-1", "vx300s_left/10_left_gripper_finger") in all_contact_pairs + or ("socket-2", "vx300s_left/10_left_gripper_finger") in all_contact_pairs + or ("socket-3", "vx300s_left/10_left_gripper_finger") in all_contact_pairs + or ("socket-4", "vx300s_left/10_left_gripper_finger") in all_contact_pairs) + + peg_touch_table = ("red_peg", "table") in all_contact_pairs + socket_touch_table = (("socket-1", "table") in all_contact_pairs or ("socket-2", "table") in all_contact_pairs + or ("socket-3", "table") in all_contact_pairs + or ("socket-4", "table") in all_contact_pairs) + peg_touch_socket = (("red_peg", "socket-1") in all_contact_pairs or ("red_peg", "socket-2") in all_contact_pairs + or ("red_peg", "socket-3") in all_contact_pairs + or ("red_peg", "socket-4") in all_contact_pairs) + pin_touched = ("red_peg", "pin") in all_contact_pairs + + reward = 0 + if touch_left_gripper and touch_right_gripper: # touch both + reward = 1 + if (touch_left_gripper and touch_right_gripper and (not peg_touch_table) + and (not socket_touch_table)): # grasp both + reward = 2 + if (peg_touch_socket and (not peg_touch_table) and (not socket_touch_table)): # peg and socket touching + reward = 3 + if pin_touched: # successful insertion + reward = 4 + return reward + + +def get_action(master_bot_left, master_bot_right): + action = np.zeros(16) + # arm action + action[:7] = master_bot_left.dxl.joint_states.position[:7] + action[8:8 + 7] = master_bot_right.dxl.joint_states.position[:7] + # gripper action + left_gripper_pos = master_bot_left.dxl.joint_states.position[8] + right_gripper_pos = master_bot_right.dxl.joint_states.position[8] + normalized_left_pos = MASTER_GRIPPER_POSITION_NORMALIZE_FN(left_gripper_pos) + normalized_right_pos = MASTER_GRIPPER_POSITION_NORMALIZE_FN(right_gripper_pos) + action[7] = normalized_left_pos + action[8 + 7] = normalized_right_pos + return action + + +def test_sim_teleop(): + """Testing teleoperation in sim with ALOHA. Requires hardware and ALOHA repo to work.""" + from interbotix_xs_modules.arm import InterbotixManipulatorXS + + BOX_POSE[0] = [0.2, 0.5, 0.05, 1, 0, 0, 0] + + # source of data + master_bot_left = InterbotixManipulatorXS( + robot_model="wx250s", + group_name="arm", + gripper_name="gripper", + robot_name=f"master_left", + init_node=True, + ) + master_bot_right = InterbotixManipulatorXS( + robot_model="wx250s", + group_name="arm", + gripper_name="gripper", + robot_name=f"master_right", + init_node=False, + ) + + # setup the environment + env = make_sim_env("sim_transfer_cube") + ts = env.reset() + episode = [ts] + # setup plotting + ax = plt.subplot() + plt_img = ax.imshow(ts.observation["images"]["angle"]) + plt.ion() + + for t in range(1000): + action = get_action(master_bot_left, master_bot_right) + ts = env.step(action) + episode.append(ts) + + plt_img.set_data(ts.observation["images"]["angle"]) + plt.pause(0.02) + + +if __name__ == "__main__": + test_sim_teleop() diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/train.sh b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/train.sh new file mode 100644 index 00000000..36efc08d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/train.sh @@ -0,0 +1,26 @@ +#!/bin/bash +task_name=${1} +task_config=${2} +expert_data_num=${3} +seed=${4} +gpu_id=${5} + +DEBUG=False +save_ckpt=True + +export CUDA_VISIBLE_DEVICES=${gpu_id} + +python3 imitate_episodes.py \ + --task_name sim-${task_name}-${task_config}-${expert_data_num} \ + --ckpt_dir ./act_ckpt/act-${task_name}/${task_config}-${expert_data_num} \ + --policy_class ACT \ + --kl_weight 10 \ + --chunk_size 50 \ + --hidden_dim 512 \ + --batch_size 8 \ + --dim_feedforward 3200 \ + --num_epochs 6000 \ + --lr 1e-5 \ + --save_freq 2000 \ + --state_dim 14 \ + --seed ${seed} diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/utils.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/utils.py new file mode 100644 index 00000000..d2e54da5 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/utils.py @@ -0,0 +1,237 @@ +import numpy as np +import torch +import os +import h5py +from torch.utils.data import TensorDataset, DataLoader + +import IPython + +e = IPython.embed + + +class EpisodicDataset(torch.utils.data.Dataset): + + def __init__(self, episode_ids, dataset_dir, camera_names, norm_stats, max_action_len): + super(EpisodicDataset).__init__() + self.episode_ids = episode_ids + self.dataset_dir = dataset_dir + self.camera_names = camera_names + self.norm_stats = norm_stats + self.max_action_len = max_action_len + self.is_sim = None + self.__getitem__(0) # initialize self.is_sim + + def __len__(self): + return len(self.episode_ids) + + def __getitem__(self, index): + sample_full_episode = False + + episode_id = self.episode_ids[index] + dataset_path = os.path.join(self.dataset_dir, f"episode_{episode_id}.hdf5") + with h5py.File(dataset_path, "r") as root: + is_sim = None + original_action_shape = root["/action"].shape + episode_len = original_action_shape[0] + if sample_full_episode: + start_ts = 0 + else: + start_ts = np.random.choice(episode_len) + # get observation at start_ts only + qpos = root["/observations/qpos"][start_ts] + image_dict = dict() + for cam_name in self.camera_names: + image_dict[cam_name] = root[f"/observations/images/{cam_name}"][start_ts] + # get all actions after and including start_ts + if is_sim: + action = root["/action"][start_ts:] + action_len = episode_len - start_ts + else: + action = root["/action"][max(0, start_ts - 1):] # hack, to make timesteps more aligned + action_len = episode_len - max(0, start_ts - 1) # hack, to make timesteps more aligned + + self.is_sim = is_sim + padded_action = np.zeros((self.max_action_len, action.shape[1]), dtype=np.float32) # 根据max_action_len初始化 + padded_action[:action_len] = action + is_pad = np.ones(self.max_action_len, dtype=bool) # 初始化为全1(True) + is_pad[:action_len] = 0 # 前action_len个位置设置为0(False),表示非填充部分 + + # new axis for different cameras + all_cam_images = [] + for cam_name in self.camera_names: + all_cam_images.append(image_dict[cam_name]) + all_cam_images = np.stack(all_cam_images, axis=0) + + # construct observations + image_data = torch.from_numpy(all_cam_images) + qpos_data = torch.from_numpy(qpos).float() + action_data = torch.from_numpy(padded_action).float() + is_pad = torch.from_numpy(is_pad).bool() + + # channel last + image_data = torch.einsum("k h w c -> k c h w", image_data) + + # normalize image and change dtype to float + image_data = image_data / 255.0 + action_data = (action_data - self.norm_stats["action_mean"]) / self.norm_stats["action_std"] + qpos_data = (qpos_data - self.norm_stats["qpos_mean"]) / self.norm_stats["qpos_std"] + + return image_data, qpos_data, action_data, is_pad + + +def get_norm_stats(dataset_dir, num_episodes): + all_qpos_data = [] + all_action_data = [] + for episode_idx in range(num_episodes): + dataset_path = os.path.join(dataset_dir, f"episode_{episode_idx}.hdf5") + with h5py.File(dataset_path, "r") as root: + qpos = root["/observations/qpos"][()] # Assuming this is a numpy array + action = root["/action"][()] + all_qpos_data.append(torch.from_numpy(qpos)) + all_action_data.append(torch.from_numpy(action)) + + # Pad all tensors to the maximum size + max_qpos_len = max(q.size(0) for q in all_qpos_data) + max_action_len = max(a.size(0) for a in all_action_data) + + padded_qpos = [] + for qpos in all_qpos_data: + current_len = qpos.size(0) + if current_len < max_qpos_len: + # Pad with the last element + pad = qpos[-1:].repeat(max_qpos_len - current_len, 1) + qpos = torch.cat([qpos, pad], dim=0) + padded_qpos.append(qpos) + + padded_action = [] + for action in all_action_data: + current_len = action.size(0) + if current_len < max_action_len: + pad = action[-1:].repeat(max_action_len - current_len, 1) + action = torch.cat([action, pad], dim=0) + padded_action.append(action) + + all_qpos_data = torch.stack(padded_qpos) + all_action_data = torch.stack(padded_action) + all_action_data = all_action_data + + # normalize action data + action_mean = all_action_data.mean(dim=[0, 1], keepdim=True) + action_std = all_action_data.std(dim=[0, 1], keepdim=True) + action_std = torch.clip(action_std, 1e-2, np.inf) # clipping + + # normalize qpos data + qpos_mean = all_qpos_data.mean(dim=[0, 1], keepdim=True) + qpos_std = all_qpos_data.std(dim=[0, 1], keepdim=True) + qpos_std = torch.clip(qpos_std, 1e-2, np.inf) # clipping + + stats = { + "action_mean": action_mean.numpy().squeeze(), + "action_std": action_std.numpy().squeeze(), + "qpos_mean": qpos_mean.numpy().squeeze(), + "qpos_std": qpos_std.numpy().squeeze(), + "example_qpos": qpos, + } + + return stats, max_action_len + + +def load_data(dataset_dir, num_episodes, camera_names, batch_size_train, batch_size_val): + print(f"\nData from: {dataset_dir}\n") + # obtain train test split + train_ratio = 0.8 + shuffled_indices = np.random.permutation(num_episodes) + train_indices = shuffled_indices[:int(train_ratio * num_episodes)] + val_indices = shuffled_indices[int(train_ratio * num_episodes):] + + # obtain normalization stats for qpos and action + norm_stats, max_action_len = get_norm_stats(dataset_dir, num_episodes) + + # construct dataset and dataloader + train_dataset = EpisodicDataset(train_indices, dataset_dir, camera_names, norm_stats, max_action_len) + val_dataset = EpisodicDataset(val_indices, dataset_dir, camera_names, norm_stats, max_action_len) + train_dataloader = DataLoader( + train_dataset, + batch_size=batch_size_train, + shuffle=True, + pin_memory=True, + num_workers=1, + prefetch_factor=1, + ) + val_dataloader = DataLoader( + val_dataset, + batch_size=batch_size_val, + shuffle=True, + pin_memory=True, + num_workers=1, + prefetch_factor=1, + ) + + return train_dataloader, val_dataloader, norm_stats, train_dataset.is_sim + + +### env utils + + +def sample_box_pose(): + x_range = [0.0, 0.2] + y_range = [0.4, 0.6] + z_range = [0.05, 0.05] + + ranges = np.vstack([x_range, y_range, z_range]) + cube_position = np.random.uniform(ranges[:, 0], ranges[:, 1]) + + cube_quat = np.array([1, 0, 0, 0]) + return np.concatenate([cube_position, cube_quat]) + + +def sample_insertion_pose(): + # Peg + x_range = [0.1, 0.2] + y_range = [0.4, 0.6] + z_range = [0.05, 0.05] + + ranges = np.vstack([x_range, y_range, z_range]) + peg_position = np.random.uniform(ranges[:, 0], ranges[:, 1]) + + peg_quat = np.array([1, 0, 0, 0]) + peg_pose = np.concatenate([peg_position, peg_quat]) + + # Socket + x_range = [-0.2, -0.1] + y_range = [0.4, 0.6] + z_range = [0.05, 0.05] + + ranges = np.vstack([x_range, y_range, z_range]) + socket_position = np.random.uniform(ranges[:, 0], ranges[:, 1]) + + socket_quat = np.array([1, 0, 0, 0]) + socket_pose = np.concatenate([socket_position, socket_quat]) + + return peg_pose, socket_pose + + +### helper functions + + +def compute_dict_mean(epoch_dicts): + result = {k: None for k in epoch_dicts[0]} + num_items = len(epoch_dicts) + for k in result: + value_sum = 0 + for epoch_dict in epoch_dicts: + value_sum += epoch_dict[k] + result[k] = value_sum / num_items + return result + + +def detach_dict(d): + new_d = dict() + for k, v in d.items(): + new_d[k] = v.detach() + return new_d + + +def set_seed(seed): + torch.manual_seed(seed) + np.random.seed(seed) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/visualize_episodes.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/visualize_episodes.py new file mode 100644 index 00000000..09e3357d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/ACT/visualize_episodes.py @@ -0,0 +1,163 @@ +import os +import numpy as np +import cv2 +import h5py +import argparse + +import matplotlib.pyplot as plt +from constants import DT + +import IPython + +e = IPython.embed + +JOINT_NAMES = [ + "waist", + "shoulder", + "elbow", + "forearm_roll", + "wrist_angle", + "wrist_rotate", +] +STATE_NAMES = JOINT_NAMES + ["gripper"] + + +def load_hdf5(dataset_dir, dataset_name): + dataset_path = os.path.join(dataset_dir, dataset_name + ".hdf5") + if not os.path.isfile(dataset_path): + print(f"Dataset does not exist at \n{dataset_path}\n") + exit() + + with h5py.File(dataset_path, "r") as root: + is_sim = root.attrs["sim"] + qpos = root["/observations/qpos"][()] + qvel = root["/observations/qvel"][()] + action = root["/action"][()] + image_dict = dict() + for cam_name in root[f"/observations/images/"].keys(): + image_dict[cam_name] = root[f"/observations/images/{cam_name}"][()] + + return qpos, qvel, action, image_dict + + +def main(args): + dataset_dir = args["dataset_dir"] + episode_idx = args["episode_idx"] + dataset_name = f"episode_{episode_idx}" + + qpos, qvel, action, image_dict = load_hdf5(dataset_dir, dataset_name) + save_videos( + image_dict, + DT, + video_path=os.path.join(dataset_dir, dataset_name + "_video.mp4"), + ) + visualize_joints(qpos, action, plot_path=os.path.join(dataset_dir, dataset_name + "_qpos.png")) + # visualize_timestamp(t_list, dataset_path) # TODO addn timestamp back + + +def save_videos(video, dt, video_path=None): + if isinstance(video, list): + cam_names = list(video[0].keys()) + h, w, _ = video[0][cam_names[0]].shape + w = w * len(cam_names) + fps = int(1 / dt) + out = cv2.VideoWriter(video_path, cv2.VideoWriter_fourcc(*"mp4v"), fps, (w, h)) + for ts, image_dict in enumerate(video): + images = [] + for cam_name in cam_names: + image = image_dict[cam_name] + image = image[:, :, [2, 1, 0]] # swap B and R channel + images.append(image) + images = np.concatenate(images, axis=1) + out.write(images) + out.release() + print(f"Saved video to: {video_path}") + elif isinstance(video, dict): + cam_names = list(video.keys()) + all_cam_videos = [] + for cam_name in cam_names: + all_cam_videos.append(video[cam_name]) + all_cam_videos = np.concatenate(all_cam_videos, axis=2) # width dimension + + n_frames, h, w, _ = all_cam_videos.shape + fps = int(1 / dt) + out = cv2.VideoWriter(video_path, cv2.VideoWriter_fourcc(*"mp4v"), fps, (w, h)) + for t in range(n_frames): + image = all_cam_videos[t] + image = image[:, :, [2, 1, 0]] # swap B and R channel + out.write(image) + out.release() + print(f"Saved video to: {video_path}") + + +def visualize_joints(qpos_list, command_list, plot_path=None, ylim=None, label_overwrite=None): + if label_overwrite: + label1, label2 = label_overwrite + else: + label1, label2 = "State", "Command" + + qpos = np.array(qpos_list) # ts, dim + command = np.array(command_list) + num_ts, num_dim = qpos.shape + h, w = 2, num_dim + num_figs = num_dim + fig, axs = plt.subplots(num_figs, 1, figsize=(w, h * num_figs)) + + # plot joint state + all_names = [name + "_left" for name in STATE_NAMES] + [name + "_right" for name in STATE_NAMES] + for dim_idx in range(num_dim): + ax = axs[dim_idx] + ax.plot(qpos[:, dim_idx], label=label1) + ax.set_title(f"Joint {dim_idx}: {all_names[dim_idx]}") + ax.legend() + + # plot arm command + for dim_idx in range(num_dim): + ax = axs[dim_idx] + ax.plot(command[:, dim_idx], label=label2) + ax.legend() + + if ylim: + for dim_idx in range(num_dim): + ax = axs[dim_idx] + ax.set_ylim(ylim) + + plt.tight_layout() + plt.savefig(plot_path) + print(f"Saved qpos plot to: {plot_path}") + plt.close() + + +def visualize_timestamp(t_list, dataset_path): + plot_path = dataset_path.replace(".pkl", "_timestamp.png") + h, w = 4, 10 + fig, axs = plt.subplots(2, 1, figsize=(w, h * 2)) + # process t_list + t_float = [] + for secs, nsecs in t_list: + t_float.append(secs + nsecs * 10e-10) + t_float = np.array(t_float) + + ax = axs[0] + ax.plot(np.arange(len(t_float)), t_float) + ax.set_title(f"Camera frame timestamps") + ax.set_xlabel("timestep") + ax.set_ylabel("time (sec)") + + ax = axs[1] + ax.plot(np.arange(len(t_float) - 1), t_float[:-1] - t_float[1:]) + ax.set_title(f"dt") + ax.set_xlabel("timestep") + ax.set_ylabel("time (sec)") + + plt.tight_layout() + plt.savefig(plot_path) + print(f"Saved timestamp plot to: {plot_path}") + plt.close() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--dataset_dir", action="store", type=str, help="Dataset dir.", required=True) + parser.add_argument("--episode_idx", action="store", type=int, help="Episode index.", required=False) + main(vars(parser.parse_args())) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/.gitignore b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/.gitignore new file mode 100644 index 00000000..0dd75ff3 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/.gitignore @@ -0,0 +1,2 @@ +data/* +checkpoints/* \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/__init__.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/__init__.py new file mode 100644 index 00000000..d4b67709 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/__init__.py @@ -0,0 +1 @@ +from .deploy_policy import * diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/deploy_policy.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/deploy_policy.py new file mode 100644 index 00000000..6e9e0c3f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/deploy_policy.py @@ -0,0 +1,51 @@ +import numpy as np +from .dp_model import DP +import yaml + +def encode_obs(observation): + head_cam = (np.moveaxis(observation["observation"]["head_camera"]["rgb"], -1, 0) / 255) + left_cam = (np.moveaxis(observation["observation"]["left_camera"]["rgb"], -1, 0) / 255) + right_cam = (np.moveaxis(observation["observation"]["right_camera"]["rgb"], -1, 0) / 255) + obs = dict( + head_cam=head_cam, + left_cam=left_cam, + right_cam=right_cam, + ) + obs["agent_pos"] = observation["joint_action"]["vector"] + return obs + + +def get_model(usr_args): + ckpt_file = f"./policy/DP/checkpoints/{usr_args['task_name']}-{usr_args['ckpt_setting']}-{usr_args['expert_data_num']}-{usr_args['seed']}/{usr_args['checkpoint_num']}.ckpt" + action_dim = usr_args['left_arm_dim'] + usr_args['right_arm_dim'] + 2 # 2 gripper + + load_config_path = f'./policy/DP/diffusion_policy/config/robot_dp_{action_dim}.yaml' + with open(load_config_path, "r", encoding="utf-8") as f: + model_training_config = yaml.safe_load(f) + + n_obs_steps = model_training_config['n_obs_steps'] + n_action_steps = model_training_config['n_action_steps'] + + return DP(ckpt_file, n_obs_steps=n_obs_steps, n_action_steps=n_action_steps) + + +def eval(TASK_ENV, model, observation): + """ + TASK_ENV: Task Environment Class, you can use this class to interact with the environment + model: The model from 'get_model()' function + observation: The observation about the environment + """ + obs = encode_obs(observation) + instruction = TASK_ENV.get_instruction() + + # ======== Get Action ======== + actions = model.get_action(obs) + + for action in actions: + TASK_ENV.take_action(action) + observation = TASK_ENV.get_obs() + obs = encode_obs(observation) + model.update_obs(obs) + +def reset_model(model): + model.reset_obs() diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/deploy_policy.yml b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/deploy_policy.yml new file mode 100644 index 00000000..14f39b16 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/deploy_policy.yml @@ -0,0 +1,11 @@ +# Basic experiment configuration +policy_name: DP +task_name: null +task_config: null +ckpt_setting: null +seed: null +instruction_type: unseen + +expert_data_num: null +checkpoint_num: 600 +head_camera_type: D435 \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/deploy_policy_double_env.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/deploy_policy_double_env.py new file mode 100644 index 00000000..762163a7 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/deploy_policy_double_env.py @@ -0,0 +1,36 @@ +import numpy as np +try: + from .dp_model import DP +except: + pass + +def encode_obs(observation): + head_cam = (np.moveaxis(observation["observation"]["head_camera"]["rgb"], -1, 0) / 255) + left_cam = (np.moveaxis(observation["observation"]["left_camera"]["rgb"], -1, 0) / 255) + right_cam = (np.moveaxis(observation["observation"]["right_camera"]["rgb"], -1, 0) / 255) + obs = dict( + head_cam=head_cam, + left_cam=left_cam, + right_cam=right_cam, + ) + obs["agent_pos"] = observation["joint_action"]["vector"] + return obs + + +def get_model(usr_args): + ckpt_file = f"./policy/DP/checkpoints/{usr_args['task_name']}-{usr_args['ckpt_setting']}-{usr_args['expert_data_num']}-{usr_args['seed']}/{usr_args['checkpoint_num']}.ckpt" + return DP(ckpt_file) + + +def eval(TASK_ENV, model, observation): + obs = encode_obs(observation) + instruction = TASK_ENV.get_instruction() + + # ======== Get Action ======== + actions = model.call(func_name='get_action', obs=obs) + + for action in actions: + TASK_ENV.take_action(action) + observation = TASK_ENV.get_obs() + obs = encode_obs(observation) + model.call(func_name='update_obs', obs=obs) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/__init__.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/checkpoint_util.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/checkpoint_util.py new file mode 100644 index 00000000..f3ce00a0 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/checkpoint_util.py @@ -0,0 +1,61 @@ +from typing import Optional, Dict +import os + + +class TopKCheckpointManager: + + def __init__( + self, + save_dir, + monitor_key: str, + mode="min", + k=1, + format_str="epoch={epoch:03d}-train_loss={train_loss:.3f}.ckpt", + ): + assert mode in ["max", "min"] + assert k >= 0 + + self.save_dir = save_dir + self.monitor_key = monitor_key + self.mode = mode + self.k = k + self.format_str = format_str + self.path_value_map = dict() + + def get_ckpt_path(self, data: Dict[str, float]) -> Optional[str]: + if self.k == 0: + return None + + value = data[self.monitor_key] + ckpt_path = os.path.join(self.save_dir, self.format_str.format(**data)) + + if len(self.path_value_map) < self.k: + # under-capacity + self.path_value_map[ckpt_path] = value + return ckpt_path + + # at capacity + sorted_map = sorted(self.path_value_map.items(), key=lambda x: x[1]) + min_path, min_value = sorted_map[0] + max_path, max_value = sorted_map[-1] + + delete_path = None + if self.mode == "max": + if value > min_value: + delete_path = min_path + else: + if value < max_value: + delete_path = max_path + + if delete_path is None: + return None + else: + del self.path_value_map[delete_path] + self.path_value_map[ckpt_path] = value + + if not os.path.exists(self.save_dir): + os.mkdir(self.save_dir) + + if os.path.exists(delete_path): + os.remove(delete_path) + return ckpt_path diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/cv2_util.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/cv2_util.py new file mode 100644 index 00000000..469ac0f8 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/cv2_util.py @@ -0,0 +1,150 @@ +from typing import Tuple +import math +import cv2 +import numpy as np + + +def draw_reticle(img, u, v, label_color): + """ + Draws a reticle (cross-hair) on the image at the given position on top of + the original image. + @param img (In/Out) uint8 3 channel image + @param u X coordinate (width) + @param v Y coordinate (height) + @param label_color tuple of 3 ints for RGB color used for drawing. + """ + # Cast to int. + u = int(u) + v = int(v) + + white = (255, 255, 255) + cv2.circle(img, (u, v), 10, label_color, 1) + cv2.circle(img, (u, v), 11, white, 1) + cv2.circle(img, (u, v), 12, label_color, 1) + cv2.line(img, (u, v + 1), (u, v + 3), white, 1) + cv2.line(img, (u + 1, v), (u + 3, v), white, 1) + cv2.line(img, (u, v - 1), (u, v - 3), white, 1) + cv2.line(img, (u - 1, v), (u - 3, v), white, 1) + + +def draw_text( + img, + *, + text, + uv_top_left, + color=(255, 255, 255), + fontScale=0.5, + thickness=1, + fontFace=cv2.FONT_HERSHEY_SIMPLEX, + outline_color=(0, 0, 0), + line_spacing=1.5, +): + """ + Draws multiline with an outline. + """ + assert isinstance(text, str) + + uv_top_left = np.array(uv_top_left, dtype=float) + assert uv_top_left.shape == (2, ) + + for line in text.splitlines(): + (w, h), _ = cv2.getTextSize( + text=line, + fontFace=fontFace, + fontScale=fontScale, + thickness=thickness, + ) + uv_bottom_left_i = uv_top_left + [0, h] + org = tuple(uv_bottom_left_i.astype(int)) + + if outline_color is not None: + cv2.putText( + img, + text=line, + org=org, + fontFace=fontFace, + fontScale=fontScale, + color=outline_color, + thickness=thickness * 3, + lineType=cv2.LINE_AA, + ) + cv2.putText( + img, + text=line, + org=org, + fontFace=fontFace, + fontScale=fontScale, + color=color, + thickness=thickness, + lineType=cv2.LINE_AA, + ) + + uv_top_left += [0, h * line_spacing] + + +def get_image_transform( + input_res: Tuple[int, int] = (1280, 720), + output_res: Tuple[int, int] = (640, 480), + bgr_to_rgb: bool = False, +): + + iw, ih = input_res + ow, oh = output_res + rw, rh = None, None + interp_method = cv2.INTER_AREA + + if (iw / ih) >= (ow / oh): + # input is wider + rh = oh + rw = math.ceil(rh / ih * iw) + if oh > ih: + interp_method = cv2.INTER_LINEAR + else: + rw = ow + rh = math.ceil(rw / iw * ih) + if ow > iw: + interp_method = cv2.INTER_LINEAR + + w_slice_start = (rw - ow) // 2 + w_slice = slice(w_slice_start, w_slice_start + ow) + h_slice_start = (rh - oh) // 2 + h_slice = slice(h_slice_start, h_slice_start + oh) + c_slice = slice(None) + if bgr_to_rgb: + c_slice = slice(None, None, -1) + + def transform(img: np.ndarray): + assert img.shape == ((ih, iw, 3)) + # resize + img = cv2.resize(img, (rw, rh), interpolation=interp_method) + # crop + img = img[h_slice, w_slice, c_slice] + return img + + return transform + + +def optimal_row_cols(n_cameras, in_wh_ratio, max_resolution=(1920, 1080)): + out_w, out_h = max_resolution + out_wh_ratio = out_w / out_h + + n_rows = np.arange(n_cameras, dtype=np.int64) + 1 + n_cols = np.ceil(n_cameras / n_rows).astype(np.int64) + cat_wh_ratio = in_wh_ratio * (n_cols / n_rows) + ratio_diff = np.abs(out_wh_ratio - cat_wh_ratio) + best_idx = np.argmin(ratio_diff) + best_n_row = n_rows[best_idx] + best_n_col = n_cols[best_idx] + best_cat_wh_ratio = cat_wh_ratio[best_idx] + + rw, rh = None, None + if best_cat_wh_ratio >= out_wh_ratio: + # cat is wider + rw = math.floor(out_w / best_n_col) + rh = math.floor(rw / in_wh_ratio) + else: + rh = math.floor(out_h / best_n_row) + rw = math.floor(rh * in_wh_ratio) + + # crop_resolution = (rw, rh) + return rw, rh, best_n_col, best_n_row diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/env_util.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/env_util.py new file mode 100644 index 00000000..30622fac --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/env_util.py @@ -0,0 +1,28 @@ +import cv2 +import numpy as np + + +def render_env_video(env, states, actions=None): + observations = states + imgs = list() + for i in range(len(observations)): + state = observations[i] + env.set_state(state) + if i == 0: + env.set_state(state) + img = env.render() + # draw action + if actions is not None: + action = actions[i] + coord = (action / 512 * 96).astype(np.int32) + cv2.drawMarker( + img, + coord, + color=(255, 0, 0), + markerType=cv2.MARKER_CROSS, + markerSize=8, + thickness=1, + ) + imgs.append(img) + imgs = np.array(imgs) + return imgs diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/json_logger.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/json_logger.py new file mode 100644 index 00000000..f4bcab79 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/json_logger.py @@ -0,0 +1,115 @@ +from typing import Optional, Callable, Any, Sequence +import os +import copy +import json +import numbers +import pandas as pd + + +def read_json_log(path: str, required_keys: Sequence[str] = tuple(), **kwargs) -> pd.DataFrame: + """ + Read json-per-line file, with potentially incomplete lines. + kwargs passed to pd.read_json + """ + lines = list() + with open(path, "r") as f: + while True: + # one json per line + line = f.readline() + if len(line) == 0: + # EOF + break + elif not line.endswith("\n"): + # incomplete line + break + is_relevant = False + for k in required_keys: + if k in line: + is_relevant = True + break + if is_relevant: + lines.append(line) + if len(lines) < 1: + return pd.DataFrame() + json_buf = (f'[{",".join([line for line in (line.strip() for line in lines) if line])}]') + df = pd.read_json(json_buf, **kwargs) + return df + + +class JsonLogger: + + def __init__(self, path: str, filter_fn: Optional[Callable[[str, Any], bool]] = None): + if filter_fn is None: + filter_fn = lambda k, v: isinstance(v, numbers.Number) + + # default to append mode + self.path = path + self.filter_fn = filter_fn + self.file = None + self.last_log = None + + def start(self): + # use line buffering + try: + self.file = file = open(self.path, "r+", buffering=1) + except FileNotFoundError: + self.file = file = open(self.path, "w+", buffering=1) + + # Move the pointer (similar to a cursor in a text editor) to the end of the file + pos = file.seek(0, os.SEEK_END) + + # Read each character in the file one at a time from the last + # character going backwards, searching for a newline character + # If we find a new line, exit the search + while pos > 0 and file.read(1) != "\n": + pos -= 1 + file.seek(pos, os.SEEK_SET) + # now the file pointer is at one past the last '\n' + # and pos is at the last '\n'. + last_line_end = file.tell() + + # find the start of second last line + pos = max(0, pos - 1) + file.seek(pos, os.SEEK_SET) + while pos > 0 and file.read(1) != "\n": + pos -= 1 + file.seek(pos, os.SEEK_SET) + # now the file pointer is at one past the second last '\n' + last_line_start = file.tell() + + if last_line_start < last_line_end: + # has last line of json + last_line = file.readline() + self.last_log = json.loads(last_line) + + # remove the last incomplete line + file.seek(last_line_end) + file.truncate() + + def stop(self): + self.file.close() + self.file = None + + def __enter__(self): + self.start() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.stop() + + def log(self, data: dict): + filtered_data = dict(filter(lambda x: self.filter_fn(*x), data.items())) + # save current as last log + self.last_log = filtered_data + for k, v in filtered_data.items(): + if isinstance(v, numbers.Integral): + filtered_data[k] = int(v) + elif isinstance(v, numbers.Number): + filtered_data[k] = float(v) + buf = json.dumps(filtered_data) + # ensure one line per json + buf = buf.replace("\n", "") + "\n" + self.file.write(buf) + + def get_last_log(self): + return copy.deepcopy(self.last_log) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/nested_dict_util.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/nested_dict_util.py new file mode 100644 index 00000000..013bd0bd --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/nested_dict_util.py @@ -0,0 +1,34 @@ +import functools + + +def nested_dict_map(f, x): + """ + Map f over all leaf of nested dict x + """ + + if not isinstance(x, dict): + return f(x) + y = dict() + for key, value in x.items(): + y[key] = nested_dict_map(f, value) + return y + + +def nested_dict_reduce(f, x): + """ + Map f over all values of nested dict x, and reduce to a single value + """ + if not isinstance(x, dict): + return x + + reduced_values = list() + for value in x.values(): + reduced_values.append(nested_dict_reduce(f, value)) + y = functools.reduce(f, reduced_values) + return y + + +def nested_dict_check(f, x): + bool_dict = nested_dict_map(f, x) + result = nested_dict_reduce(lambda x, y: x and y, bool_dict) + return result diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/normalize_util.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/normalize_util.py new file mode 100644 index 00000000..0b0e3f15 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/normalize_util.py @@ -0,0 +1,197 @@ +from diffusion_policy.model.common.normalizer import SingleFieldLinearNormalizer +from diffusion_policy.common.pytorch_util import ( + dict_apply, + dict_apply_reduce, + dict_apply_split, +) +import numpy as np + + +def get_range_normalizer_from_stat(stat, output_max=1, output_min=-1, range_eps=1e-7): + # -1, 1 normalization + input_max = stat["max"] + input_min = stat["min"] + input_range = input_max - input_min + ignore_dim = input_range < range_eps + input_range[ignore_dim] = output_max - output_min + scale = (output_max - output_min) / input_range + offset = output_min - scale * input_min + offset[ignore_dim] = (output_max + output_min) / 2 - input_min[ignore_dim] + + return SingleFieldLinearNormalizer.create_manual(scale=scale, offset=offset, input_stats_dict=stat) + + +def get_image_range_normalizer(): + scale = np.array([2], dtype=np.float32) + offset = np.array([-1], dtype=np.float32) + stat = { + "min": np.array([0], dtype=np.float32), + "max": np.array([1], dtype=np.float32), + "mean": np.array([0.5], dtype=np.float32), + "std": np.array([np.sqrt(1 / 12)], dtype=np.float32), + } + return SingleFieldLinearNormalizer.create_manual(scale=scale, offset=offset, input_stats_dict=stat) + + +def get_identity_normalizer_from_stat(stat): + scale = np.ones_like(stat["min"]) + offset = np.zeros_like(stat["min"]) + return SingleFieldLinearNormalizer.create_manual(scale=scale, offset=offset, input_stats_dict=stat) + + +def robomimic_abs_action_normalizer_from_stat(stat, rotation_transformer): + result = dict_apply_split(stat, lambda x: {"pos": x[..., :3], "rot": x[..., 3:6], "gripper": x[..., 6:]}) + + def get_pos_param_info(stat, output_max=1, output_min=-1, range_eps=1e-7): + # -1, 1 normalization + input_max = stat["max"] + input_min = stat["min"] + input_range = input_max - input_min + ignore_dim = input_range < range_eps + input_range[ignore_dim] = output_max - output_min + scale = (output_max - output_min) / input_range + offset = output_min - scale * input_min + offset[ignore_dim] = (output_max + output_min) / 2 - input_min[ignore_dim] + + return {"scale": scale, "offset": offset}, stat + + def get_rot_param_info(stat): + example = rotation_transformer.forward(stat["mean"]) + scale = np.ones_like(example) + offset = np.zeros_like(example) + info = { + "max": np.ones_like(example), + "min": np.full_like(example, -1), + "mean": np.zeros_like(example), + "std": np.ones_like(example), + } + return {"scale": scale, "offset": offset}, info + + def get_gripper_param_info(stat): + example = stat["max"] + scale = np.ones_like(example) + offset = np.zeros_like(example) + info = { + "max": np.ones_like(example), + "min": np.full_like(example, -1), + "mean": np.zeros_like(example), + "std": np.ones_like(example), + } + return {"scale": scale, "offset": offset}, info + + pos_param, pos_info = get_pos_param_info(result["pos"]) + rot_param, rot_info = get_rot_param_info(result["rot"]) + gripper_param, gripper_info = get_gripper_param_info(result["gripper"]) + + param = dict_apply_reduce([pos_param, rot_param, gripper_param], lambda x: np.concatenate(x, axis=-1)) + info = dict_apply_reduce([pos_info, rot_info, gripper_info], lambda x: np.concatenate(x, axis=-1)) + + return SingleFieldLinearNormalizer.create_manual(scale=param["scale"], + offset=param["offset"], + input_stats_dict=info) + + +def robomimic_abs_action_only_normalizer_from_stat(stat): + result = dict_apply_split(stat, lambda x: {"pos": x[..., :3], "other": x[..., 3:]}) + + def get_pos_param_info(stat, output_max=1, output_min=-1, range_eps=1e-7): + # -1, 1 normalization + input_max = stat["max"] + input_min = stat["min"] + input_range = input_max - input_min + ignore_dim = input_range < range_eps + input_range[ignore_dim] = output_max - output_min + scale = (output_max - output_min) / input_range + offset = output_min - scale * input_min + offset[ignore_dim] = (output_max + output_min) / 2 - input_min[ignore_dim] + + return {"scale": scale, "offset": offset}, stat + + def get_other_param_info(stat): + example = stat["max"] + scale = np.ones_like(example) + offset = np.zeros_like(example) + info = { + "max": np.ones_like(example), + "min": np.full_like(example, -1), + "mean": np.zeros_like(example), + "std": np.ones_like(example), + } + return {"scale": scale, "offset": offset}, info + + pos_param, pos_info = get_pos_param_info(result["pos"]) + other_param, other_info = get_other_param_info(result["other"]) + + param = dict_apply_reduce([pos_param, other_param], lambda x: np.concatenate(x, axis=-1)) + info = dict_apply_reduce([pos_info, other_info], lambda x: np.concatenate(x, axis=-1)) + + return SingleFieldLinearNormalizer.create_manual(scale=param["scale"], + offset=param["offset"], + input_stats_dict=info) + + +def robomimic_abs_action_only_dual_arm_normalizer_from_stat(stat): + Da = stat["max"].shape[-1] + Dah = Da // 2 + result = dict_apply_split( + stat, + lambda x: { + "pos0": x[..., :3], + "other0": x[..., 3:Dah], + "pos1": x[..., Dah:Dah + 3], + "other1": x[..., Dah + 3:], + }, + ) + + def get_pos_param_info(stat, output_max=1, output_min=-1, range_eps=1e-7): + # -1, 1 normalization + input_max = stat["max"] + input_min = stat["min"] + input_range = input_max - input_min + ignore_dim = input_range < range_eps + input_range[ignore_dim] = output_max - output_min + scale = (output_max - output_min) / input_range + offset = output_min - scale * input_min + offset[ignore_dim] = (output_max + output_min) / 2 - input_min[ignore_dim] + + return {"scale": scale, "offset": offset}, stat + + def get_other_param_info(stat): + example = stat["max"] + scale = np.ones_like(example) + offset = np.zeros_like(example) + info = { + "max": np.ones_like(example), + "min": np.full_like(example, -1), + "mean": np.zeros_like(example), + "std": np.ones_like(example), + } + return {"scale": scale, "offset": offset}, info + + pos0_param, pos0_info = get_pos_param_info(result["pos0"]) + pos1_param, pos1_info = get_pos_param_info(result["pos1"]) + other0_param, other0_info = get_other_param_info(result["other0"]) + other1_param, other1_info = get_other_param_info(result["other1"]) + + param = dict_apply_reduce( + [pos0_param, other0_param, pos1_param, other1_param], + lambda x: np.concatenate(x, axis=-1), + ) + info = dict_apply_reduce( + [pos0_info, other0_info, pos1_info, other1_info], + lambda x: np.concatenate(x, axis=-1), + ) + + return SingleFieldLinearNormalizer.create_manual(scale=param["scale"], + offset=param["offset"], + input_stats_dict=info) + + +def array_to_stats(arr: np.ndarray): + stat = { + "min": np.min(arr, axis=0), + "max": np.max(arr, axis=0), + "mean": np.mean(arr, axis=0), + "std": np.std(arr, axis=0), + } + return stat diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/pose_trajectory_interpolator.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/pose_trajectory_interpolator.py new file mode 100644 index 00000000..091aca9b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/pose_trajectory_interpolator.py @@ -0,0 +1,211 @@ +from typing import Union +import numbers +import numpy as np +import scipy.interpolate as si +import scipy.spatial.transform as st + + +def rotation_distance(a: st.Rotation, b: st.Rotation) -> float: + return (b * a.inv()).magnitude() + + +def pose_distance(start_pose, end_pose): + start_pose = np.array(start_pose) + end_pose = np.array(end_pose) + start_pos = start_pose[:3] + end_pos = end_pose[:3] + start_rot = st.Rotation.from_rotvec(start_pose[3:]) + end_rot = st.Rotation.from_rotvec(end_pose[3:]) + pos_dist = np.linalg.norm(end_pos - start_pos) + rot_dist = rotation_distance(start_rot, end_rot) + return pos_dist, rot_dist + + +class PoseTrajectoryInterpolator: + + def __init__(self, times: np.ndarray, poses: np.ndarray): + assert len(times) >= 1 + assert len(poses) == len(times) + if not isinstance(times, np.ndarray): + times = np.array(times) + if not isinstance(poses, np.ndarray): + poses = np.array(poses) + + if len(times) == 1: + # special treatment for single step interpolation + self.single_step = True + self._times = times + self._poses = poses + else: + self.single_step = False + assert np.all(times[1:] >= times[:-1]) + + pos = poses[:, :3] + rot = st.Rotation.from_rotvec(poses[:, 3:]) + + self.pos_interp = si.interp1d(times, pos, axis=0, assume_sorted=True) + self.rot_interp = st.Slerp(times, rot) + + @property + def times(self) -> np.ndarray: + if self.single_step: + return self._times + else: + return self.pos_interp.x + + @property + def poses(self) -> np.ndarray: + if self.single_step: + return self._poses + else: + n = len(self.times) + poses = np.zeros((n, 6)) + poses[:, :3] = self.pos_interp.y + poses[:, 3:] = self.rot_interp(self.times).as_rotvec() + return poses + + def trim(self, start_t: float, end_t: float) -> "PoseTrajectoryInterpolator": + assert start_t <= end_t + times = self.times + should_keep = (start_t < times) & (times < end_t) + keep_times = times[should_keep] + all_times = np.concatenate([[start_t], keep_times, [end_t]]) + # remove duplicates, Slerp requires strictly increasing x + all_times = np.unique(all_times) + # interpolate + all_poses = self(all_times) + return PoseTrajectoryInterpolator(times=all_times, poses=all_poses) + + def drive_to_waypoint(self, + pose, + time, + curr_time, + max_pos_speed=np.inf, + max_rot_speed=np.inf) -> "PoseTrajectoryInterpolator": + assert max_pos_speed > 0 + assert max_rot_speed > 0 + time = max(time, curr_time) + + curr_pose = self(curr_time) + pos_dist, rot_dist = pose_distance(curr_pose, pose) + pos_min_duration = pos_dist / max_pos_speed + rot_min_duration = rot_dist / max_rot_speed + duration = time - curr_time + duration = max(duration, max(pos_min_duration, rot_min_duration)) + assert duration >= 0 + last_waypoint_time = curr_time + duration + + # insert new pose + trimmed_interp = self.trim(curr_time, curr_time) + times = np.append(trimmed_interp.times, [last_waypoint_time], axis=0) + poses = np.append(trimmed_interp.poses, [pose], axis=0) + + # create new interpolator + final_interp = PoseTrajectoryInterpolator(times, poses) + return final_interp + + def schedule_waypoint( + self, + pose, + time, + max_pos_speed=np.inf, + max_rot_speed=np.inf, + curr_time=None, + last_waypoint_time=None, + ) -> "PoseTrajectoryInterpolator": + assert max_pos_speed > 0 + assert max_rot_speed > 0 + if last_waypoint_time is not None: + assert curr_time is not None + + # trim current interpolator to between curr_time and last_waypoint_time + start_time = self.times[0] + end_time = self.times[-1] + assert start_time <= end_time + + if curr_time is not None: + if time <= curr_time: + # if insert time is earlier than current time + # no effect should be done to the interpolator + return self + # now, curr_time < time + start_time = max(curr_time, start_time) + + if last_waypoint_time is not None: + # if last_waypoint_time is earlier than start_time + # use start_time + if time <= last_waypoint_time: + end_time = curr_time + else: + end_time = max(last_waypoint_time, curr_time) + else: + end_time = curr_time + + end_time = min(end_time, time) + start_time = min(start_time, end_time) + # end time should be the latest of all times except time + # after this we can assume order (proven by zhenjia, due to the 2 min operations) + + # Constraints: + # start_time <= end_time <= time (proven by zhenjia) + # curr_time <= start_time (proven by zhenjia) + # curr_time <= time (proven by zhenjia) + + # time can't change + # last_waypoint_time can't change + # curr_time can't change + assert start_time <= end_time + assert end_time <= time + if last_waypoint_time is not None: + if time <= last_waypoint_time: + assert end_time == curr_time + else: + assert end_time == max(last_waypoint_time, curr_time) + + if curr_time is not None: + assert curr_time <= start_time + assert curr_time <= time + + trimmed_interp = self.trim(start_time, end_time) + # after this, all waypoints in trimmed_interp is within start_time and end_time + # and is earlier than time + + # determine speed + duration = time - end_time + end_pose = trimmed_interp(end_time) + pos_dist, rot_dist = pose_distance(pose, end_pose) + pos_min_duration = pos_dist / max_pos_speed + rot_min_duration = rot_dist / max_rot_speed + duration = max(duration, max(pos_min_duration, rot_min_duration)) + assert duration >= 0 + last_waypoint_time = end_time + duration + + # insert new pose + times = np.append(trimmed_interp.times, [last_waypoint_time], axis=0) + poses = np.append(trimmed_interp.poses, [pose], axis=0) + + # create new interpolator + final_interp = PoseTrajectoryInterpolator(times, poses) + return final_interp + + def __call__(self, t: Union[numbers.Number, np.ndarray]) -> np.ndarray: + is_single = False + if isinstance(t, numbers.Number): + is_single = True + t = np.array([t]) + + pose = np.zeros((len(t), 6)) + if self.single_step: + pose[:] = self._poses[0] + else: + start_time = self.times[0] + end_time = self.times[-1] + t = np.clip(t, start_time, end_time) + + pose = np.zeros((len(t), 6)) + pose[:, :3] = self.pos_interp(t) + pose[:, 3:] = self.rot_interp(t).as_rotvec() + + if is_single: + pose = pose[0] + return pose diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/precise_sleep.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/precise_sleep.py new file mode 100644 index 00000000..04d783ee --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/precise_sleep.py @@ -0,0 +1,27 @@ +import time + + +def precise_sleep(dt: float, slack_time: float = 0.001, time_func=time.monotonic): + """ + Use hybrid of time.sleep and spinning to minimize jitter. + Sleep dt - slack_time seconds first, then spin for the rest. + """ + t_start = time_func() + if dt > slack_time: + time.sleep(dt - slack_time) + t_end = t_start + dt + while time_func() < t_end: + pass + return + + +def precise_wait(t_end: float, slack_time: float = 0.001, time_func=time.monotonic): + t_start = time_func() + t_wait = t_end - t_start + if t_wait > 0: + t_sleep = t_wait - slack_time + if t_sleep > 0: + time.sleep(t_sleep) + while time_func() < t_end: + pass + return diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/pymunk_override.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/pymunk_override.py new file mode 100644 index 00000000..1c85f868 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/pymunk_override.py @@ -0,0 +1,246 @@ +# ---------------------------------------------------------------------------- +# pymunk +# Copyright (c) 2007-2016 Victor Blomqvist +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# ---------------------------------------------------------------------------- +"""This submodule contains helper functions to help with quick prototyping +using pymunk together with pygame. + +Intended to help with debugging and prototyping, not for actual production use +in a full application. The methods contained in this module is opinionated +about your coordinate system and not in any way optimized. +""" + +__docformat__ = "reStructuredText" + +__all__ = [ + "DrawOptions", + "get_mouse_pos", + "to_pygame", + "from_pygame", + "lighten", + "positive_y_is_up", +] + +from typing import List, Sequence, Tuple + +import pygame + +import numpy as np + +import pymunk +from pymunk.space_debug_draw_options import SpaceDebugColor +from pymunk.vec2d import Vec2d + +positive_y_is_up: bool = False +"""Make increasing values of y point upwards. + +When True:: + + y + ^ + | . (3, 3) + | + | . (2, 2) + | + +------ > x + +When False:: + + +------ > x + | + | . (2, 2) + | + | . (3, 3) + v + y + +""" + + +class DrawOptions(pymunk.SpaceDebugDrawOptions): + + def __init__(self, surface: pygame.Surface) -> None: + """Draw a pymunk.Space on a pygame.Surface object. + + Typical usage:: + + >>> import pymunk + >>> surface = pygame.Surface((10,10)) + >>> space = pymunk.Space() + >>> options = pymunk.pygame_util.DrawOptions(surface) + >>> space.debug_draw(options) + + You can control the color of a shape by setting shape.color to the color + you want it drawn in:: + + >>> c = pymunk.Circle(None, 10) + >>> c.color = pygame.Color("pink") + + See pygame_util.demo.py for a full example + + Since pygame uses a coordinate system where y points down (in contrast + to many other cases), you either have to make the physics simulation + with Pymunk also behave in that way, or flip everything when you draw. + + The easiest is probably to just make the simulation behave the same + way as Pygame does. In that way all coordinates used are in the same + orientation and easy to reason about:: + + >>> space = pymunk.Space() + >>> space.gravity = (0, -1000) + >>> body = pymunk.Body() + >>> body.position = (0, 0) # will be positioned in the top left corner + >>> space.debug_draw(options) + + To flip the drawing its possible to set the module property + :py:data:`positive_y_is_up` to True. Then the pygame drawing will flip + the simulation upside down before drawing:: + + >>> positive_y_is_up = True + >>> body = pymunk.Body() + >>> body.position = (0, 0) + >>> # Body will be position in bottom left corner + + :Parameters: + surface : pygame.Surface + Surface that the objects will be drawn on + """ + self.surface = surface + super(DrawOptions, self).__init__() + + def draw_circle( + self, + pos: Vec2d, + angle: float, + radius: float, + outline_color: SpaceDebugColor, + fill_color: SpaceDebugColor, + ) -> None: + p = to_pygame(pos, self.surface) + + pygame.draw.circle(self.surface, fill_color.as_int(), p, round(radius), 0) + pygame.draw.circle(self.surface, light_color(fill_color).as_int(), p, round(radius - 4), 0) + + circle_edge = pos + Vec2d(radius, 0).rotated(angle) + p2 = to_pygame(circle_edge, self.surface) + line_r = 2 if radius > 20 else 1 + # pygame.draw.lines(self.surface, outline_color.as_int(), False, [p, p2], line_r) + + def draw_segment(self, a: Vec2d, b: Vec2d, color: SpaceDebugColor) -> None: + p1 = to_pygame(a, self.surface) + p2 = to_pygame(b, self.surface) + + pygame.draw.aalines(self.surface, color.as_int(), False, [p1, p2]) + + def draw_fat_segment( + self, + a: Tuple[float, float], + b: Tuple[float, float], + radius: float, + outline_color: SpaceDebugColor, + fill_color: SpaceDebugColor, + ) -> None: + p1 = to_pygame(a, self.surface) + p2 = to_pygame(b, self.surface) + + r = round(max(1, radius * 2)) + pygame.draw.lines(self.surface, fill_color.as_int(), False, [p1, p2], r) + if r > 2: + orthog = [abs(p2[1] - p1[1]), abs(p2[0] - p1[0])] + if orthog[0] == 0 and orthog[1] == 0: + return + scale = radius / (orthog[0] * orthog[0] + orthog[1] * orthog[1])**0.5 + orthog[0] = round(orthog[0] * scale) + orthog[1] = round(orthog[1] * scale) + points = [ + (p1[0] - orthog[0], p1[1] - orthog[1]), + (p1[0] + orthog[0], p1[1] + orthog[1]), + (p2[0] + orthog[0], p2[1] + orthog[1]), + (p2[0] - orthog[0], p2[1] - orthog[1]), + ] + pygame.draw.polygon(self.surface, fill_color.as_int(), points) + pygame.draw.circle( + self.surface, + fill_color.as_int(), + (round(p1[0]), round(p1[1])), + round(radius), + ) + pygame.draw.circle( + self.surface, + fill_color.as_int(), + (round(p2[0]), round(p2[1])), + round(radius), + ) + + def draw_polygon( + self, + verts: Sequence[Tuple[float, float]], + radius: float, + outline_color: SpaceDebugColor, + fill_color: SpaceDebugColor, + ) -> None: + ps = [to_pygame(v, self.surface) for v in verts] + ps += [ps[0]] + + radius = 2 + pygame.draw.polygon(self.surface, light_color(fill_color).as_int(), ps) + + if radius > 0: + for i in range(len(verts)): + a = verts[i] + b = verts[(i + 1) % len(verts)] + self.draw_fat_segment(a, b, radius, fill_color, fill_color) + + def draw_dot(self, size: float, pos: Tuple[float, float], color: SpaceDebugColor) -> None: + p = to_pygame(pos, self.surface) + pygame.draw.circle(self.surface, color.as_int(), p, round(size), 0) + + +def get_mouse_pos(surface: pygame.Surface) -> Tuple[int, int]: + """Get position of the mouse pointer in pymunk coordinates.""" + p = pygame.mouse.get_pos() + return from_pygame(p, surface) + + +def to_pygame(p: Tuple[float, float], surface: pygame.Surface) -> Tuple[int, int]: + """Convenience method to convert pymunk coordinates to pygame surface + local coordinates. + + Note that in case positive_y_is_up is False, this function won't actually do + anything except converting the point to integers. + """ + if positive_y_is_up: + return round(p[0]), surface.get_height() - round(p[1]) + else: + return round(p[0]), round(p[1]) + + +def from_pygame(p: Tuple[float, float], surface: pygame.Surface) -> Tuple[int, int]: + """Convenience method to convert pygame surface local coordinates to + pymunk coordinates + """ + return to_pygame(p, surface) + + +def light_color(color: SpaceDebugColor): + color = np.minimum(1.2 * np.float32([color.r, color.g, color.b, color.a]), np.float32([255])) + color = SpaceDebugColor(r=color[0], g=color[1], b=color[2], a=color[3]) + return color diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/pymunk_util.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/pymunk_util.py new file mode 100644 index 00000000..5f279e6f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/pymunk_util.py @@ -0,0 +1,51 @@ +import pygame +import pymunk +import pymunk.pygame_util +import numpy as np + +COLLTYPE_DEFAULT = 0 +COLLTYPE_MOUSE = 1 +COLLTYPE_BALL = 2 + + +def get_body_type(static=False): + body_type = pymunk.Body.DYNAMIC + if static: + body_type = pymunk.Body.STATIC + return body_type + + +def create_rectangle(space, pos_x, pos_y, width, height, density=3, static=False): + body = pymunk.Body(body_type=get_body_type(static)) + body.position = (pos_x, pos_y) + shape = pymunk.Poly.create_box(body, (width, height)) + shape.density = density + space.add(body, shape) + return body, shape + + +def create_rectangle_bb(space, left, bottom, right, top, **kwargs): + pos_x = (left + right) / 2 + pos_y = (top + bottom) / 2 + height = top - bottom + width = right - left + return create_rectangle(space, pos_x, pos_y, width, height, **kwargs) + + +def create_circle(space, pos_x, pos_y, radius, density=3, static=False): + body = pymunk.Body(body_type=get_body_type(static)) + body.position = (pos_x, pos_y) + shape = pymunk.Circle(body, radius=radius) + shape.density = density + shape.collision_type = COLLTYPE_BALL + space.add(body, shape) + return body, shape + + +def get_body_state(body): + state = np.zeros(6, dtype=np.float32) + state[:2] = body.position + state[2] = body.angle + state[3:5] = body.velocity + state[5] = body.angular_velocity + return state diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/pytorch_util.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/pytorch_util.py new file mode 100644 index 00000000..a4fe51ae --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/pytorch_util.py @@ -0,0 +1,81 @@ +from typing import Dict, Callable, List +import collections +import torch +import torch.nn as nn + + +def dict_apply(x: Dict[str, torch.Tensor], func: Callable[[torch.Tensor], torch.Tensor]) -> Dict[str, torch.Tensor]: + result = dict() + for key, value in x.items(): + if isinstance(value, dict): + result[key] = dict_apply(value, func) + else: + result[key] = func(value) + return result + + +def pad_remaining_dims(x, target): + assert x.shape == target.shape[:len(x.shape)] + return x.reshape(x.shape + (1, ) * (len(target.shape) - len(x.shape))) + + +def dict_apply_split( + x: Dict[str, torch.Tensor], + split_func: Callable[[torch.Tensor], Dict[str, torch.Tensor]], +) -> Dict[str, torch.Tensor]: + results = collections.defaultdict(dict) + for key, value in x.items(): + result = split_func(value) + for k, v in result.items(): + results[k][key] = v + return results + + +def dict_apply_reduce( + x: List[Dict[str, torch.Tensor]], + reduce_func: Callable[[List[torch.Tensor]], torch.Tensor], +) -> Dict[str, torch.Tensor]: + result = dict() + for key in x[0].keys(): + result[key] = reduce_func([x_[key] for x_ in x]) + return result + + +def replace_submodules( + root_module: nn.Module, + predicate: Callable[[nn.Module], bool], + func: Callable[[nn.Module], nn.Module], +) -> nn.Module: + """ + predicate: Return true if the module is to be replaced. + func: Return new module to use. + """ + if predicate(root_module): + return func(root_module) + + bn_list = [k.split(".") for k, m in root_module.named_modules(remove_duplicate=True) if predicate(m)] + for *parent, k in bn_list: + parent_module = root_module + if len(parent) > 0: + parent_module = root_module.get_submodule(".".join(parent)) + if isinstance(parent_module, nn.Sequential): + src_module = parent_module[int(k)] + else: + src_module = getattr(parent_module, k) + tgt_module = func(src_module) + if isinstance(parent_module, nn.Sequential): + parent_module[int(k)] = tgt_module + else: + setattr(parent_module, k, tgt_module) + # verify that all BN are replaced + bn_list = [k.split(".") for k, m in root_module.named_modules(remove_duplicate=True) if predicate(m)] + assert len(bn_list) == 0 + return root_module + + +def optimizer_to(optimizer, device): + for state in optimizer.state.values(): + for k, v in state.items(): + if isinstance(v, torch.Tensor): + state[k] = v.to(device=device) + return optimizer diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/replay_buffer.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/replay_buffer.py new file mode 100644 index 00000000..344a540b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/replay_buffer.py @@ -0,0 +1,622 @@ +from typing import Union, Dict, Optional +import os +import math +import numbers +import zarr +import numcodecs +import numpy as np +from functools import cached_property + + +def check_chunks_compatible(chunks: tuple, shape: tuple): + assert len(shape) == len(chunks) + for c in chunks: + assert isinstance(c, numbers.Integral) + assert c > 0 + + +def rechunk_recompress_array(group, name, chunks=None, chunk_length=None, compressor=None, tmp_key="_temp"): + old_arr = group[name] + if chunks is None: + if chunk_length is not None: + chunks = (chunk_length, ) + old_arr.chunks[1:] + else: + chunks = old_arr.chunks + check_chunks_compatible(chunks, old_arr.shape) + + if compressor is None: + compressor = old_arr.compressor + + if (chunks == old_arr.chunks) and (compressor == old_arr.compressor): + # no change + return old_arr + + # rechunk recompress + group.move(name, tmp_key) + old_arr = group[tmp_key] + n_copied, n_skipped, n_bytes_copied = zarr.copy( + source=old_arr, + dest=group, + name=name, + chunks=chunks, + compressor=compressor, + ) + del group[tmp_key] + arr = group[name] + return arr + + +def get_optimal_chunks(shape, dtype, target_chunk_bytes=2e6, max_chunk_length=None): + """ + Common shapes + T,D + T,N,D + T,H,W,C + T,N,H,W,C + """ + itemsize = np.dtype(dtype).itemsize + # reversed + rshape = list(shape[::-1]) + if max_chunk_length is not None: + rshape[-1] = int(max_chunk_length) + split_idx = len(shape) - 1 + for i in range(len(shape) - 1): + this_chunk_bytes = itemsize * np.prod(rshape[:i]) + next_chunk_bytes = itemsize * np.prod(rshape[:i + 1]) + if (this_chunk_bytes <= target_chunk_bytes and next_chunk_bytes > target_chunk_bytes): + split_idx = i + + rchunks = rshape[:split_idx] + item_chunk_bytes = itemsize * np.prod(rshape[:split_idx]) + this_max_chunk_length = rshape[split_idx] + next_chunk_length = min(this_max_chunk_length, math.ceil(target_chunk_bytes / item_chunk_bytes)) + rchunks.append(next_chunk_length) + len_diff = len(shape) - len(rchunks) + rchunks.extend([1] * len_diff) + chunks = tuple(rchunks[::-1]) + # print(np.prod(chunks) * itemsize / target_chunk_bytes) + return chunks + + +class ReplayBuffer: + """ + Zarr-based temporal datastructure. + Assumes first dimension to be time. Only chunk in time dimension. + """ + + def __init__(self, root: Union[zarr.Group, Dict[str, dict]]): + """ + Dummy constructor. Use copy_from* and create_from* class methods instead. + """ + assert "data" in root + assert "meta" in root + assert "episode_ends" in root["meta"] + for key, value in root["data"].items(): + assert value.shape[0] == root["meta"]["episode_ends"][-1] + self.root = root + + # ============= create constructors =============== + @classmethod + def create_empty_zarr(cls, storage=None, root=None): + if root is None: + if storage is None: + storage = zarr.MemoryStore() + root = zarr.group(store=storage) + data = root.require_group("data", overwrite=False) + meta = root.require_group("meta", overwrite=False) + if "episode_ends" not in meta: + episode_ends = meta.zeros( + "episode_ends", + shape=(0, ), + dtype=np.int64, + compressor=None, + overwrite=False, + ) + return cls(root=root) + + @classmethod + def create_empty_numpy(cls): + root = { + "data": dict(), + "meta": { + "episode_ends": np.zeros((0, ), dtype=np.int64) + }, + } + return cls(root=root) + + @classmethod + def create_from_group(cls, group, **kwargs): + if "data" not in group: + # create from stratch + buffer = cls.create_empty_zarr(root=group, **kwargs) + else: + # already exist + buffer = cls(root=group, **kwargs) + return buffer + + @classmethod + def create_from_path(cls, zarr_path, mode="r", **kwargs): + """ + Open a on-disk zarr directly (for dataset larger than memory). + Slower. + """ + group = zarr.open(os.path.expanduser(zarr_path), mode) + return cls.create_from_group(group, **kwargs) + + # ============= copy constructors =============== + @classmethod + def copy_from_store( + cls, + src_store, + store=None, + keys=None, + chunks: Dict[str, tuple] = dict(), + compressors: Union[dict, str, numcodecs.abc.Codec] = dict(), + if_exists="replace", + **kwargs, + ): + """ + Load to memory. + """ + src_root = zarr.group(src_store) + root = None + if store is None: + # numpy backend + meta = dict() + for key, value in src_root["meta"].items(): + if len(value.shape) == 0: + meta[key] = np.array(value) + else: + meta[key] = value[:] + + if keys is None: + keys = src_root["data"].keys() + data = dict() + for key in keys: + arr = src_root["data"][key] + data[key] = arr[:] + + root = {"meta": meta, "data": data} + else: + root = zarr.group(store=store) + # copy without recompression + n_copied, n_skipped, n_bytes_copied = zarr.copy_store( + source=src_store, + dest=store, + source_path="/meta", + dest_path="/meta", + if_exists=if_exists, + ) + data_group = root.create_group("data", overwrite=True) + if keys is None: + keys = src_root["data"].keys() + for key in keys: + value = src_root["data"][key] + cks = cls._resolve_array_chunks(chunks=chunks, key=key, array=value) + cpr = cls._resolve_array_compressor(compressors=compressors, key=key, array=value) + if cks == value.chunks and cpr == value.compressor: + # copy without recompression + this_path = "/data/" + key + n_copied, n_skipped, n_bytes_copied = zarr.copy_store( + source=src_store, + dest=store, + source_path=this_path, + dest_path=this_path, + if_exists=if_exists, + ) + else: + # copy with recompression + n_copied, n_skipped, n_bytes_copied = zarr.copy( + source=value, + dest=data_group, + name=key, + chunks=cks, + compressor=cpr, + if_exists=if_exists, + ) + buffer = cls(root=root) + return buffer + + @classmethod + def copy_from_path( + cls, + zarr_path, + backend=None, + store=None, + keys=None, + chunks: Dict[str, tuple] = dict(), + compressors: Union[dict, str, numcodecs.abc.Codec] = dict(), + if_exists="replace", + **kwargs, + ): + """ + Copy a on-disk zarr to in-memory compressed. + Recommended + """ + if backend == "numpy": + print("backend argument is deprecated!") + store = None + group = zarr.open(os.path.expanduser(zarr_path), "r") + return cls.copy_from_store( + src_store=group.store, + store=store, + keys=keys, + chunks=chunks, + compressors=compressors, + if_exists=if_exists, + **kwargs, + ) + + # ============= save methods =============== + def save_to_store( + self, + store, + chunks: Optional[Dict[str, tuple]] = dict(), + compressors: Union[str, numcodecs.abc.Codec, dict] = dict(), + if_exists="replace", + **kwargs, + ): + + root = zarr.group(store) + if self.backend == "zarr": + # recompression free copy + n_copied, n_skipped, n_bytes_copied = zarr.copy_store( + source=self.root.store, + dest=store, + source_path="/meta", + dest_path="/meta", + if_exists=if_exists, + ) + else: + meta_group = root.create_group("meta", overwrite=True) + # save meta, no chunking + for key, value in self.root["meta"].items(): + _ = meta_group.array(name=key, data=value, shape=value.shape, chunks=value.shape) + + # save data, chunk + data_group = root.create_group("data", overwrite=True) + for key, value in self.root["data"].items(): + cks = self._resolve_array_chunks(chunks=chunks, key=key, array=value) + cpr = self._resolve_array_compressor(compressors=compressors, key=key, array=value) + if isinstance(value, zarr.Array): + if cks == value.chunks and cpr == value.compressor: + # copy without recompression + this_path = "/data/" + key + n_copied, n_skipped, n_bytes_copied = zarr.copy_store( + source=self.root.store, + dest=store, + source_path=this_path, + dest_path=this_path, + if_exists=if_exists, + ) + else: + # copy with recompression + n_copied, n_skipped, n_bytes_copied = zarr.copy( + source=value, + dest=data_group, + name=key, + chunks=cks, + compressor=cpr, + if_exists=if_exists, + ) + else: + # numpy + _ = data_group.array(name=key, data=value, chunks=cks, compressor=cpr) + return store + + def save_to_path( + self, + zarr_path, + chunks: Optional[Dict[str, tuple]] = dict(), + compressors: Union[str, numcodecs.abc.Codec, dict] = dict(), + if_exists="replace", + **kwargs, + ): + store = zarr.DirectoryStore(os.path.expanduser(zarr_path)) + return self.save_to_store(store, chunks=chunks, compressors=compressors, if_exists=if_exists, **kwargs) + + @staticmethod + def resolve_compressor(compressor="default"): + if compressor == "default": + compressor = numcodecs.Blosc(cname="lz4", clevel=5, shuffle=numcodecs.Blosc.NOSHUFFLE) + elif compressor == "disk": + compressor = numcodecs.Blosc("zstd", clevel=5, shuffle=numcodecs.Blosc.BITSHUFFLE) + return compressor + + @classmethod + def _resolve_array_compressor(cls, compressors: Union[dict, str, numcodecs.abc.Codec], key, array): + # allows compressor to be explicitly set to None + cpr = "nil" + if isinstance(compressors, dict): + if key in compressors: + cpr = cls.resolve_compressor(compressors[key]) + elif isinstance(array, zarr.Array): + cpr = array.compressor + else: + cpr = cls.resolve_compressor(compressors) + # backup default + if cpr == "nil": + cpr = cls.resolve_compressor("default") + return cpr + + @classmethod + def _resolve_array_chunks(cls, chunks: Union[dict, tuple], key, array): + cks = None + if isinstance(chunks, dict): + if key in chunks: + cks = chunks[key] + elif isinstance(array, zarr.Array): + cks = array.chunks + elif isinstance(chunks, tuple): + cks = chunks + else: + raise TypeError(f"Unsupported chunks type {type(chunks)}") + # backup default + if cks is None: + cks = get_optimal_chunks(shape=array.shape, dtype=array.dtype) + # check + check_chunks_compatible(chunks=cks, shape=array.shape) + return cks + + # ============= properties ================= + @cached_property + def data(self): + return self.root["data"] + + @cached_property + def meta(self): + return self.root["meta"] + + def update_meta(self, data): + # sanitize data + np_data = dict() + for key, value in data.items(): + if isinstance(value, np.ndarray): + np_data[key] = value + else: + arr = np.array(value) + if arr.dtype == object: + raise TypeError(f"Invalid value type {type(value)}") + np_data[key] = arr + + meta_group = self.meta + if self.backend == "zarr": + for key, value in np_data.items(): + _ = meta_group.array( + name=key, + data=value, + shape=value.shape, + chunks=value.shape, + overwrite=True, + ) + else: + meta_group.update(np_data) + + return meta_group + + @property + def episode_ends(self): + return self.meta["episode_ends"] + + def get_episode_idxs(self): + import numba + + numba.jit(nopython=True) + + def _get_episode_idxs(episode_ends): + result = np.zeros((episode_ends[-1], ), dtype=np.int64) + for i in range(len(episode_ends)): + start = 0 + if i > 0: + start = episode_ends[i - 1] + end = episode_ends[i] + for idx in range(start, end): + result[idx] = i + return result + + return _get_episode_idxs(self.episode_ends) + + @property + def backend(self): + backend = "numpy" + if isinstance(self.root, zarr.Group): + backend = "zarr" + return backend + + # =========== dict-like API ============== + def __repr__(self) -> str: + if self.backend == "zarr": + return str(self.root.tree()) + else: + return super().__repr__() + + def keys(self): + return self.data.keys() + + def values(self): + return self.data.values() + + def items(self): + return self.data.items() + + def __getitem__(self, key): + return self.data[key] + + def __contains__(self, key): + return key in self.data + + # =========== our API ============== + @property + def n_steps(self): + if len(self.episode_ends) == 0: + return 0 + return self.episode_ends[-1] + + @property + def n_episodes(self): + return len(self.episode_ends) + + @property + def chunk_size(self): + if self.backend == "zarr": + return next(iter(self.data.arrays()))[-1].chunks[0] + return None + + @property + def episode_lengths(self): + ends = self.episode_ends[:] + ends = np.insert(ends, 0, 0) + lengths = np.diff(ends) + return lengths + + def add_episode( + self, + data: Dict[str, np.ndarray], + chunks: Optional[Dict[str, tuple]] = dict(), + compressors: Union[str, numcodecs.abc.Codec, dict] = dict(), + ): + assert len(data) > 0 + is_zarr = self.backend == "zarr" + + curr_len = self.n_steps + episode_length = None + for key, value in data.items(): + assert len(value.shape) >= 1 + if episode_length is None: + episode_length = len(value) + else: + assert episode_length == len(value) + new_len = curr_len + episode_length + + for key, value in data.items(): + new_shape = (new_len, ) + value.shape[1:] + # create array + if key not in self.data: + if is_zarr: + cks = self._resolve_array_chunks(chunks=chunks, key=key, array=value) + cpr = self._resolve_array_compressor(compressors=compressors, key=key, array=value) + arr = self.data.zeros( + name=key, + shape=new_shape, + chunks=cks, + dtype=value.dtype, + compressor=cpr, + ) + else: + # copy data to prevent modify + arr = np.zeros(shape=new_shape, dtype=value.dtype) + self.data[key] = arr + else: + arr = self.data[key] + assert value.shape[1:] == arr.shape[1:] + # same method for both zarr and numpy + if is_zarr: + arr.resize(new_shape) + else: + arr.resize(new_shape, refcheck=False) + # copy data + arr[-value.shape[0]:] = value + + # append to episode ends + episode_ends = self.episode_ends + if is_zarr: + episode_ends.resize(episode_ends.shape[0] + 1) + else: + episode_ends.resize(episode_ends.shape[0] + 1, refcheck=False) + episode_ends[-1] = new_len + + # rechunk + if is_zarr: + if episode_ends.chunks[0] < episode_ends.shape[0]: + rechunk_recompress_array( + self.meta, + "episode_ends", + chunk_length=int(episode_ends.shape[0] * 1.5), + ) + + def drop_episode(self): + is_zarr = self.backend == "zarr" + episode_ends = self.episode_ends[:].copy() + assert len(episode_ends) > 0 + start_idx = 0 + if len(episode_ends) > 1: + start_idx = episode_ends[-2] + for key, value in self.data.items(): + new_shape = (start_idx, ) + value.shape[1:] + if is_zarr: + value.resize(new_shape) + else: + value.resize(new_shape, refcheck=False) + if is_zarr: + self.episode_ends.resize(len(episode_ends) - 1) + else: + self.episode_ends.resize(len(episode_ends) - 1, refcheck=False) + + def pop_episode(self): + assert self.n_episodes > 0 + episode = self.get_episode(self.n_episodes - 1, copy=True) + self.drop_episode() + return episode + + def extend(self, data): + self.add_episode(data) + + def get_episode(self, idx, copy=False): + idx = list(range(len(self.episode_ends)))[idx] + start_idx = 0 + if idx > 0: + start_idx = self.episode_ends[idx - 1] + end_idx = self.episode_ends[idx] + result = self.get_steps_slice(start_idx, end_idx, copy=copy) + return result + + def get_episode_slice(self, idx): + start_idx = 0 + if idx > 0: + start_idx = self.episode_ends[idx - 1] + end_idx = self.episode_ends[idx] + return slice(start_idx, end_idx) + + def get_steps_slice(self, start, stop, step=None, copy=False): + _slice = slice(start, stop, step) + + result = dict() + for key, value in self.data.items(): + x = value[_slice] + if copy and isinstance(value, np.ndarray): + x = x.copy() + result[key] = x + return result + + # =========== chunking ============= + def get_chunks(self) -> dict: + assert self.backend == "zarr" + chunks = dict() + for key, value in self.data.items(): + chunks[key] = value.chunks + return chunks + + def set_chunks(self, chunks: dict): + assert self.backend == "zarr" + for key, value in chunks.items(): + if key in self.data: + arr = self.data[key] + if value != arr.chunks: + check_chunks_compatible(chunks=value, shape=arr.shape) + rechunk_recompress_array(self.data, key, chunks=value) + + def get_compressors(self) -> dict: + assert self.backend == "zarr" + compressors = dict() + for key, value in self.data.items(): + compressors[key] = value.compressor + return compressors + + def set_compressors(self, compressors: dict): + assert self.backend == "zarr" + for key, value in compressors.items(): + if key in self.data: + arr = self.data[key] + compressor = self.resolve_compressor(value) + if compressor != arr.compressor: + rechunk_recompress_array(self.data, key, compressor=compressor) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/robomimic_config_util.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/robomimic_config_util.py new file mode 100644 index 00000000..633f4f88 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/robomimic_config_util.py @@ -0,0 +1,41 @@ +from omegaconf import OmegaConf +from robomimic.config import config_factory +import robomimic.scripts.generate_paper_configs as gpc +from robomimic.scripts.generate_paper_configs import ( + modify_config_for_default_image_exp, + modify_config_for_default_low_dim_exp, + modify_config_for_dataset, +) + + +def get_robomimic_config(algo_name="bc_rnn", hdf5_type="low_dim", task_name="square", dataset_type="ph"): + base_dataset_dir = "/tmp/null" + filter_key = None + + # decide whether to use low-dim or image training defaults + modifier_for_obs = modify_config_for_default_image_exp + if hdf5_type in ["low_dim", "low_dim_sparse", "low_dim_dense"]: + modifier_for_obs = modify_config_for_default_low_dim_exp + + algo_config_name = "bc" if algo_name == "bc_rnn" else algo_name + config = config_factory(algo_name=algo_config_name) + # turn into default config for observation modalities (e.g.: low-dim or rgb) + config = modifier_for_obs(config) + # add in config based on the dataset + config = modify_config_for_dataset( + config=config, + task_name=task_name, + dataset_type=dataset_type, + hdf5_type=hdf5_type, + base_dataset_dir=base_dataset_dir, + filter_key=filter_key, + ) + # add in algo hypers based on dataset + algo_config_modifier = getattr(gpc, f"modify_{algo_name}_config_for_dataset") + config = algo_config_modifier( + config=config, + task_name=task_name, + dataset_type=dataset_type, + hdf5_type=hdf5_type, + ) + return config diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/robomimic_util.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/robomimic_util.py new file mode 100644 index 00000000..c655172e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/robomimic_util.py @@ -0,0 +1,170 @@ +import numpy as np +import copy + +import h5py +import robomimic.utils.obs_utils as ObsUtils +import robomimic.utils.file_utils as FileUtils +import robomimic.utils.env_utils as EnvUtils +from scipy.spatial.transform import Rotation + +from robomimic.config import config_factory + + +class RobomimicAbsoluteActionConverter: + + def __init__(self, dataset_path, algo_name="bc"): + # default BC config + config = config_factory(algo_name=algo_name) + + # read config to set up metadata for observation modalities (e.g. detecting rgb observations) + # must ran before create dataset + ObsUtils.initialize_obs_utils_with_config(config) + + env_meta = FileUtils.get_env_metadata_from_dataset(dataset_path) + abs_env_meta = copy.deepcopy(env_meta) + abs_env_meta["env_kwargs"]["controller_configs"]["control_delta"] = False + + env = EnvUtils.create_env_from_metadata( + env_meta=env_meta, + render=False, + render_offscreen=False, + use_image_obs=False, + ) + assert len(env.env.robots) in (1, 2) + + abs_env = EnvUtils.create_env_from_metadata( + env_meta=abs_env_meta, + render=False, + render_offscreen=False, + use_image_obs=False, + ) + assert not abs_env.env.robots[0].controller.use_delta + + self.env = env + self.abs_env = abs_env + self.file = h5py.File(dataset_path, "r") + + def __len__(self): + return len(self.file["data"]) + + def convert_actions(self, states: np.ndarray, actions: np.ndarray) -> np.ndarray: + """ + Given state and delta action sequence + generate equivalent goal position and orientation for each step + keep the original gripper action intact. + """ + # in case of multi robot + # reshape (N,14) to (N,2,7) + # or (N,7) to (N,1,7) + stacked_actions = actions.reshape(*actions.shape[:-1], -1, 7) + + env = self.env + # generate abs actions + action_goal_pos = np.zeros(stacked_actions.shape[:-1] + (3, ), dtype=stacked_actions.dtype) + action_goal_ori = np.zeros(stacked_actions.shape[:-1] + (3, ), dtype=stacked_actions.dtype) + action_gripper = stacked_actions[..., [-1]] + for i in range(len(states)): + _ = env.reset_to({"states": states[i]}) + + # taken from robot_env.py L#454 + for idx, robot in enumerate(env.env.robots): + # run controller goal generator + robot.control(stacked_actions[i, idx], policy_step=True) + + # read pos and ori from robots + controller = robot.controller + action_goal_pos[i, idx] = controller.goal_pos + action_goal_ori[i, idx] = Rotation.from_matrix(controller.goal_ori).as_rotvec() + + stacked_abs_actions = np.concatenate([action_goal_pos, action_goal_ori, action_gripper], axis=-1) + abs_actions = stacked_abs_actions.reshape(actions.shape) + return abs_actions + + def convert_idx(self, idx): + file = self.file + demo = file[f"data/demo_{idx}"] + # input + states = demo["states"][:] + actions = demo["actions"][:] + + # generate abs actions + abs_actions = self.convert_actions(states, actions) + return abs_actions + + def convert_and_eval_idx(self, idx): + env = self.env + abs_env = self.abs_env + file = self.file + # first step have high error for some reason, not representative + eval_skip_steps = 1 + + demo = file[f"data/demo_{idx}"] + # input + states = demo["states"][:] + actions = demo["actions"][:] + + # generate abs actions + abs_actions = self.convert_actions(states, actions) + + # verify + robot0_eef_pos = demo["obs"]["robot0_eef_pos"][:] + robot0_eef_quat = demo["obs"]["robot0_eef_quat"][:] + + delta_error_info = self.evaluate_rollout_error( + env, + states, + actions, + robot0_eef_pos, + robot0_eef_quat, + metric_skip_steps=eval_skip_steps, + ) + abs_error_info = self.evaluate_rollout_error( + abs_env, + states, + abs_actions, + robot0_eef_pos, + robot0_eef_quat, + metric_skip_steps=eval_skip_steps, + ) + + info = {"delta_max_error": delta_error_info, "abs_max_error": abs_error_info} + return abs_actions, info + + @staticmethod + def evaluate_rollout_error(env, states, actions, robot0_eef_pos, robot0_eef_quat, metric_skip_steps=1): + # first step have high error for some reason, not representative + + # evaluate abs actions + rollout_next_states = list() + rollout_next_eef_pos = list() + rollout_next_eef_quat = list() + obs = env.reset_to({"states": states[0]}) + for i in range(len(states)): + obs = env.reset_to({"states": states[i]}) + obs, reward, done, info = env.step(actions[i]) + obs = env.get_observation() + rollout_next_states.append(env.get_state()["states"]) + rollout_next_eef_pos.append(obs["robot0_eef_pos"]) + rollout_next_eef_quat.append(obs["robot0_eef_quat"]) + rollout_next_states = np.array(rollout_next_states) + rollout_next_eef_pos = np.array(rollout_next_eef_pos) + rollout_next_eef_quat = np.array(rollout_next_eef_quat) + + next_state_diff = states[1:] - rollout_next_states[:-1] + max_next_state_diff = np.max(np.abs(next_state_diff[metric_skip_steps:])) + + next_eef_pos_diff = robot0_eef_pos[1:] - rollout_next_eef_pos[:-1] + next_eef_pos_dist = np.linalg.norm(next_eef_pos_diff, axis=-1) + max_next_eef_pos_dist = next_eef_pos_dist[metric_skip_steps:].max() + + next_eef_rot_diff = (Rotation.from_quat(robot0_eef_quat[1:]) * + Rotation.from_quat(rollout_next_eef_quat[:-1]).inv()) + next_eef_rot_dist = next_eef_rot_diff.magnitude() + max_next_eef_rot_dist = next_eef_rot_dist[metric_skip_steps:].max() + + info = { + "state": max_next_state_diff, + "pos": max_next_eef_pos_dist, + "rot": max_next_eef_rot_dist, + } + return info diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/sampler.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/sampler.py new file mode 100644 index 00000000..85116177 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/sampler.py @@ -0,0 +1,164 @@ +from typing import Optional +import numpy as np +import numba +from diffusion_policy.common.replay_buffer import ReplayBuffer + + +@numba.jit(nopython=True) +def create_indices( + episode_ends: np.ndarray, + sequence_length: int, + episode_mask: np.ndarray, + pad_before: int = 0, + pad_after: int = 0, + debug: bool = True, +) -> np.ndarray: + episode_mask.shape == episode_ends.shape + pad_before = min(max(pad_before, 0), sequence_length - 1) + pad_after = min(max(pad_after, 0), sequence_length - 1) + + indices = list() + for i in range(len(episode_ends)): + if not episode_mask[i]: + # skip episode + continue + start_idx = 0 + if i > 0: + start_idx = episode_ends[i - 1] + end_idx = episode_ends[i] + episode_length = end_idx - start_idx + + min_start = -pad_before + max_start = episode_length - sequence_length + pad_after + + # range stops one idx before end + for idx in range(min_start, max_start + 1): + buffer_start_idx = max(idx, 0) + start_idx + buffer_end_idx = min(idx + sequence_length, episode_length) + start_idx + start_offset = buffer_start_idx - (idx + start_idx) + end_offset = (idx + sequence_length + start_idx) - buffer_end_idx + sample_start_idx = 0 + start_offset + sample_end_idx = sequence_length - end_offset + if debug: + assert start_offset >= 0 + assert end_offset >= 0 + assert (sample_end_idx - sample_start_idx) == (buffer_end_idx - buffer_start_idx) + indices.append([buffer_start_idx, buffer_end_idx, sample_start_idx, sample_end_idx]) + indices = np.array(indices) + return indices + + +def get_val_mask(n_episodes, val_ratio, seed=0): + val_mask = np.zeros(n_episodes, dtype=bool) + if val_ratio <= 0: + return val_mask + + # have at least 1 episode for validation, and at least 1 episode for train + n_val = min(max(1, round(n_episodes * val_ratio)), n_episodes - 1) + rng = np.random.default_rng(seed=seed) + # val_idxs = rng.choice(n_episodes, size=n_val, replace=False) + val_idxs = -1 + val_mask[val_idxs] = True + return val_mask + + +def downsample_mask(mask, max_n, seed=0): + # subsample training data + train_mask = mask + if (max_n is not None) and (np.sum(train_mask) > max_n): + n_train = int(max_n) + curr_train_idxs = np.nonzero(train_mask)[0] + rng = np.random.default_rng(seed=seed) + train_idxs_idx = rng.choice(len(curr_train_idxs), size=n_train, replace=False) + train_idxs = curr_train_idxs[train_idxs_idx] + train_mask = np.zeros_like(train_mask) + train_mask[train_idxs] = True + assert np.sum(train_mask) == n_train + return train_mask + + +class SequenceSampler: + + def __init__( + self, + replay_buffer: ReplayBuffer, + sequence_length: int, + pad_before: int = 0, + pad_after: int = 0, + keys=None, + key_first_k=dict(), + episode_mask: Optional[np.ndarray] = None, + ): + """ + key_first_k: dict str: int + Only take first k data from these keys (to improve perf) + """ + + super().__init__() + assert sequence_length >= 1 + if keys is None: + keys = list(replay_buffer.keys()) + + episode_ends = replay_buffer.episode_ends[:] + if episode_mask is None: + episode_mask = np.ones(episode_ends.shape, dtype=bool) + + if np.any(episode_mask): + indices = create_indices( + episode_ends, + sequence_length=sequence_length, + pad_before=pad_before, + pad_after=pad_after, + episode_mask=episode_mask, + ) + else: + indices = np.zeros((0, 4), dtype=np.int64) + + # (buffer_start_idx, buffer_end_idx, sample_start_idx, sample_end_idx) + self.indices = indices + self.keys = list(keys) # prevent OmegaConf list performance problem + self.sequence_length = sequence_length + self.replay_buffer = replay_buffer + self.key_first_k = key_first_k + + def __len__(self): + return len(self.indices) + + def sample_sequence(self, idx): + buffer_start_idx, buffer_end_idx, sample_start_idx, sample_end_idx = (self.indices[idx]) + result = dict() + for key in self.keys: + input_arr = self.replay_buffer[key] + # performance optimization, avoid small allocation if possible + if key not in self.key_first_k: + sample = input_arr[buffer_start_idx:buffer_end_idx] + else: + # performance optimization, only load used obs steps + n_data = buffer_end_idx - buffer_start_idx + k_data = min(self.key_first_k[key], n_data) + # fill value with Nan to catch bugs + # the non-loaded region should never be used + sample = np.full( + (n_data, ) + input_arr.shape[1:], + fill_value=np.nan, + dtype=input_arr.dtype, + ) + try: + sample[:k_data] = input_arr[buffer_start_idx:buffer_start_idx + k_data] + except Exception as e: + import pdb + + pdb.set_trace() + data = sample + if (sample_start_idx > 0) or (sample_end_idx < self.sequence_length): + data = np.zeros( + shape=(self.sequence_length, ) + input_arr.shape[1:], + dtype=input_arr.dtype, + ) + if sample_start_idx > 0: + data[:sample_start_idx] = sample[0] + if sample_end_idx < self.sequence_length: + data[sample_end_idx:] = sample[-1] + data[sample_start_idx:sample_end_idx] = sample + result[key] = data + return result diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/timestamp_accumulator.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/timestamp_accumulator.py new file mode 100644 index 00000000..b3be2974 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/common/timestamp_accumulator.py @@ -0,0 +1,220 @@ +from typing import List, Tuple, Optional, Dict +import math +import numpy as np + + +def get_accumulate_timestamp_idxs( + timestamps: List[float], + start_time: float, + dt: float, + eps: float = 1e-5, + next_global_idx: Optional[int] = 0, + allow_negative=False, +) -> Tuple[List[int], List[int], int]: + """ + For each dt window, choose the first timestamp in the window. + Assumes timestamps sorted. One timestamp might be chosen multiple times due to dropped frames. + next_global_idx should start at 0 normally, and then use the returned next_global_idx. + However, when overwiting previous values are desired, set last_global_idx to None. + + Returns: + local_idxs: which index in the given timestamps array to chose from + global_idxs: the global index of each chosen timestamp + next_global_idx: used for next call. + """ + local_idxs = list() + global_idxs = list() + for local_idx, ts in enumerate(timestamps): + # add eps * dt to timestamps so that when ts == start_time + k * dt + # is always recorded as kth element (avoiding floating point errors) + global_idx = math.floor((ts - start_time) / dt + eps) + if (not allow_negative) and (global_idx < 0): + continue + if next_global_idx is None: + next_global_idx = global_idx + + n_repeats = max(0, global_idx - next_global_idx + 1) + for i in range(n_repeats): + local_idxs.append(local_idx) + global_idxs.append(next_global_idx + i) + next_global_idx += n_repeats + return local_idxs, global_idxs, next_global_idx + + +def align_timestamps( + timestamps: List[float], + target_global_idxs: List[int], + start_time: float, + dt: float, + eps: float = 1e-5, +): + if isinstance(target_global_idxs, np.ndarray): + target_global_idxs = target_global_idxs.tolist() + assert len(target_global_idxs) > 0 + + local_idxs, global_idxs, _ = get_accumulate_timestamp_idxs( + timestamps=timestamps, + start_time=start_time, + dt=dt, + eps=eps, + next_global_idx=target_global_idxs[0], + allow_negative=True, + ) + if len(global_idxs) > len(target_global_idxs): + # if more steps available, truncate + global_idxs = global_idxs[:len(target_global_idxs)] + local_idxs = local_idxs[:len(target_global_idxs)] + + if len(global_idxs) == 0: + import pdb + + pdb.set_trace() + + for i in range(len(target_global_idxs) - len(global_idxs)): + # if missing, repeat + local_idxs.append(len(timestamps) - 1) + global_idxs.append(global_idxs[-1] + 1) + assert global_idxs == target_global_idxs + assert len(local_idxs) == len(global_idxs) + return local_idxs + + +class TimestampObsAccumulator: + + def __init__(self, start_time: float, dt: float, eps: float = 1e-5): + self.start_time = start_time + self.dt = dt + self.eps = eps + self.obs_buffer = dict() + self.timestamp_buffer = None + self.next_global_idx = 0 + + def __len__(self): + return self.next_global_idx + + @property + def data(self): + if self.timestamp_buffer is None: + return dict() + result = dict() + for key, value in self.obs_buffer.items(): + result[key] = value[:len(self)] + return result + + @property + def actual_timestamps(self): + if self.timestamp_buffer is None: + return np.array([]) + return self.timestamp_buffer[:len(self)] + + @property + def timestamps(self): + if self.timestamp_buffer is None: + return np.array([]) + return self.start_time + np.arange(len(self)) * self.dt + + def put(self, data: Dict[str, np.ndarray], timestamps: np.ndarray): + """ + data: + key: T,* + """ + + local_idxs, global_idxs, self.next_global_idx = get_accumulate_timestamp_idxs( + timestamps=timestamps, + start_time=self.start_time, + dt=self.dt, + eps=self.eps, + next_global_idx=self.next_global_idx, + ) + + if len(global_idxs) > 0: + if self.timestamp_buffer is None: + # first allocation + self.obs_buffer = dict() + for key, value in data.items(): + self.obs_buffer[key] = np.zeros_like(value) + self.timestamp_buffer = np.zeros((len(timestamps), ), dtype=np.float64) + + this_max_size = global_idxs[-1] + 1 + if this_max_size > len(self.timestamp_buffer): + # reallocate + new_size = max(this_max_size, len(self.timestamp_buffer) * 2) + for key in list(self.obs_buffer.keys()): + new_shape = (new_size, ) + self.obs_buffer[key].shape[1:] + self.obs_buffer[key] = np.resize(self.obs_buffer[key], new_shape) + self.timestamp_buffer = np.resize(self.timestamp_buffer, (new_size)) + + # write data + for key, value in self.obs_buffer.items(): + value[global_idxs] = data[key][local_idxs] + self.timestamp_buffer[global_idxs] = timestamps[local_idxs] + + +class TimestampActionAccumulator: + + def __init__(self, start_time: float, dt: float, eps: float = 1e-5): + """ + Different from Obs accumulator, the action accumulator + allows overwriting previous values. + """ + self.start_time = start_time + self.dt = dt + self.eps = eps + self.action_buffer = None + self.timestamp_buffer = None + self.size = 0 + + def __len__(self): + return self.size + + @property + def actions(self): + if self.action_buffer is None: + return np.array([]) + return self.action_buffer[:len(self)] + + @property + def actual_timestamps(self): + if self.timestamp_buffer is None: + return np.array([]) + return self.timestamp_buffer[:len(self)] + + @property + def timestamps(self): + if self.timestamp_buffer is None: + return np.array([]) + return self.start_time + np.arange(len(self)) * self.dt + + def put(self, actions: np.ndarray, timestamps: np.ndarray): + """ + Note: timestamps is the time when the action will be issued, + not when the action will be completed (target_timestamp) + """ + + local_idxs, global_idxs, _ = get_accumulate_timestamp_idxs( + timestamps=timestamps, + start_time=self.start_time, + dt=self.dt, + eps=self.eps, + # allows overwriting previous actions + next_global_idx=None, + ) + + if len(global_idxs) > 0: + if self.timestamp_buffer is None: + # first allocation + self.action_buffer = np.zeros_like(actions) + self.timestamp_buffer = np.zeros((len(actions), ), dtype=np.float64) + + this_max_size = global_idxs[-1] + 1 + if this_max_size > len(self.timestamp_buffer): + # reallocate + new_size = max(this_max_size, len(self.timestamp_buffer) * 2) + new_shape = (new_size, ) + self.action_buffer.shape[1:] + self.action_buffer = np.resize(self.action_buffer, new_shape) + self.timestamp_buffer = np.resize(self.timestamp_buffer, (new_size, )) + + # potentially rewrite old data (as expected) + self.action_buffer[global_idxs] = actions[local_idxs] + self.timestamp_buffer[global_idxs] = timestamps[local_idxs] + self.size = max(self.size, this_max_size) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/config/robot_dp_14.yaml b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/config/robot_dp_14.yaml new file mode 100644 index 00000000..7f8cde99 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/config/robot_dp_14.yaml @@ -0,0 +1,155 @@ +defaults: + - _self_ + - task: default_task_14 + +name: robot_${task.name} +_target_: diffusion_policy.workspace.robotworkspace.RobotWorkspace + +task_name: ${task.name} +shape_meta: ${task.shape_meta} +exp_name: "default" + +horizon: 8 +n_obs_steps: 3 +n_action_steps: 6 +n_latency_steps: 0 +dataset_obs_steps: ${n_obs_steps} +past_action_visible: False +keypoint_visible_rate: 1.0 +obs_as_global_cond: True + +policy: + _target_: diffusion_policy.policy.diffusion_unet_image_policy.DiffusionUnetImagePolicy + + shape_meta: ${shape_meta} + + noise_scheduler: + _target_: diffusers.schedulers.scheduling_ddpm.DDPMScheduler + num_train_timesteps: 100 + beta_start: 0.0001 + beta_end: 0.02 + beta_schedule: squaredcos_cap_v2 + variance_type: fixed_small # Yilun's paper uses fixed_small_log instead, but easy to cause Nan + clip_sample: True # required when predict_epsilon=False + prediction_type: epsilon # or sample + + obs_encoder: + _target_: diffusion_policy.model.vision.multi_image_obs_encoder.MultiImageObsEncoder + shape_meta: ${shape_meta} + rgb_model: + _target_: diffusion_policy.model.vision.model_getter.get_resnet + name: resnet18 + weights: null + resize_shape: null + crop_shape: null + # constant center crop + random_crop: True + use_group_norm: True + share_rgb_model: False + imagenet_norm: True + + horizon: ${horizon} + n_action_steps: ${eval:'${n_action_steps}+${n_latency_steps}'} + n_obs_steps: ${n_obs_steps} + num_inference_steps: 100 + obs_as_global_cond: ${obs_as_global_cond} + # crop_shape: null + diffusion_step_embed_dim: 128 + # down_dims: [512, 1024, 2048] + down_dims: [256, 512, 1024] + kernel_size: 5 + n_groups: 8 + cond_predict_scale: True + + # scheduler.step params + # predict_epsilon: True + +ema: + _target_: diffusion_policy.model.diffusion.ema_model.EMAModel + update_after_step: 0 + inv_gamma: 1.0 + power: 0.75 + min_value: 0.0 + max_value: 0.9999 + +dataloader: + batch_size: 128 + num_workers: 0 + shuffle: True + pin_memory: True + persistent_workers: False + +val_dataloader: + batch_size: 128 + num_workers: 0 + shuffle: False + pin_memory: True + persistent_workers: False + +optimizer: + _target_: torch.optim.AdamW + lr: 1.0e-4 + betas: [0.95, 0.999] + eps: 1.0e-8 + weight_decay: 1.0e-6 + +training: + device: "cuda:0" + seed: 42 + debug: False + resume: True + # optimization + lr_scheduler: cosine + lr_warmup_steps: 500 + num_epochs: 600 + gradient_accumulate_every: 1 + # EMA destroys performance when used with BatchNorm + # replace BatchNorm with GroupNorm. + use_ema: True + freeze_encoder: False + # training loop control + # in epochs + rollout_every: 50 + checkpoint_every: 300 + val_every: 1 + sample_every: 5 + # steps per epoch + max_train_steps: null + max_val_steps: null + # misc + tqdm_interval_sec: 1.0 + +logging: + project: diffusion_policy_debug + resume: True + mode: online + name: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name} + tags: ["${name}", "${task_name}", "${exp_name}"] + id: null + group: null + +checkpoint: + topk: + monitor_key: test_mean_score + mode: max + k: 5 + format_str: 'epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt' + save_last_ckpt: True + save_last_snapshot: False + +multi_run: + run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name} + wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name} + +hydra: + job: + override_dirname: ${name} + run: + dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name} + sweep: + dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name} + subdir: ${hydra.job.num} + +setting: null +expert_data_num: null +head_camera_type: null \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/config/robot_dp_16.yaml b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/config/robot_dp_16.yaml new file mode 100644 index 00000000..4ea76f55 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/config/robot_dp_16.yaml @@ -0,0 +1,155 @@ +defaults: + - _self_ + - task: default_task_16 + +name: robot_${task.name} +_target_: diffusion_policy.workspace.robotworkspace.RobotWorkspace + +task_name: ${task.name} +shape_meta: ${task.shape_meta} +exp_name: "default" + +horizon: 8 +n_obs_steps: 3 +n_action_steps: 6 +n_latency_steps: 0 +dataset_obs_steps: ${n_obs_steps} +past_action_visible: False +keypoint_visible_rate: 1.0 +obs_as_global_cond: True + +policy: + _target_: diffusion_policy.policy.diffusion_unet_image_policy.DiffusionUnetImagePolicy + + shape_meta: ${shape_meta} + + noise_scheduler: + _target_: diffusers.schedulers.scheduling_ddpm.DDPMScheduler + num_train_timesteps: 100 + beta_start: 0.0001 + beta_end: 0.02 + beta_schedule: squaredcos_cap_v2 + variance_type: fixed_small # Yilun's paper uses fixed_small_log instead, but easy to cause Nan + clip_sample: True # required when predict_epsilon=False + prediction_type: epsilon # or sample + + obs_encoder: + _target_: diffusion_policy.model.vision.multi_image_obs_encoder.MultiImageObsEncoder + shape_meta: ${shape_meta} + rgb_model: + _target_: diffusion_policy.model.vision.model_getter.get_resnet + name: resnet18 + weights: null + resize_shape: null + crop_shape: null + # constant center crop + random_crop: True + use_group_norm: True + share_rgb_model: False + imagenet_norm: True + + horizon: ${horizon} + n_action_steps: ${eval:'${n_action_steps}+${n_latency_steps}'} + n_obs_steps: ${n_obs_steps} + num_inference_steps: 100 + obs_as_global_cond: ${obs_as_global_cond} + # crop_shape: null + diffusion_step_embed_dim: 128 + # down_dims: [512, 1024, 2048] + down_dims: [256, 512, 1024] + kernel_size: 5 + n_groups: 8 + cond_predict_scale: True + + # scheduler.step params + # predict_epsilon: True + +ema: + _target_: diffusion_policy.model.diffusion.ema_model.EMAModel + update_after_step: 0 + inv_gamma: 1.0 + power: 0.75 + min_value: 0.0 + max_value: 0.9999 + +dataloader: + batch_size: 128 + num_workers: 0 + shuffle: True + pin_memory: True + persistent_workers: False + +val_dataloader: + batch_size: 128 + num_workers: 0 + shuffle: False + pin_memory: True + persistent_workers: False + +optimizer: + _target_: torch.optim.AdamW + lr: 1.0e-4 + betas: [0.95, 0.999] + eps: 1.0e-8 + weight_decay: 1.0e-6 + +training: + device: "cuda:0" + seed: 42 + debug: False + resume: True + # optimization + lr_scheduler: cosine + lr_warmup_steps: 500 + num_epochs: 600 + gradient_accumulate_every: 1 + # EMA destroys performance when used with BatchNorm + # replace BatchNorm with GroupNorm. + use_ema: True + freeze_encoder: False + # training loop control + # in epochs + rollout_every: 50 + checkpoint_every: 300 + val_every: 1 + sample_every: 5 + # steps per epoch + max_train_steps: null + max_val_steps: null + # misc + tqdm_interval_sec: 1.0 + +logging: + project: diffusion_policy_debug + resume: True + mode: online + name: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name} + tags: ["${name}", "${task_name}", "${exp_name}"] + id: null + group: null + +checkpoint: + topk: + monitor_key: test_mean_score + mode: max + k: 5 + format_str: 'epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt' + save_last_ckpt: True + save_last_snapshot: False + +multi_run: + run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name} + wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name} + +hydra: + job: + override_dirname: ${name} + run: + dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name} + sweep: + dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name} + subdir: ${hydra.job.num} + +setting: null +expert_data_num: null +head_camera_type: null \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/config/task/default_task_14.yaml b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/config/task/default_task_14.yaml new file mode 100644 index 00000000..c9d5a480 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/config/task/default_task_14.yaml @@ -0,0 +1,50 @@ +name: task_config + +image_shape: &image_shape [3, -1, -1] +shape_meta: &shape_meta + # acceptable types: rgb, low_dim + obs: + head_cam: + shape: *image_shape + type: rgb + # front_cam: + # shape: *image_shape + # type: rgb + # left_cam: + # shape: *image_shape + # type: rgb + # right_cam: + # shape: *image_shape + # type: rgb + agent_pos: + shape: [14] + type: low_dim + action: + shape: [14] + +env_runner: + _target_: diffusion_policy.env_runner.pusht_image_runner.PushTImageRunner + n_train: 6 + n_train_vis: 2 + train_start_seed: 0 + n_test: 50 + n_test_vis: 4 + legacy_test: True + test_start_seed: 100000 + max_steps: 300 + n_obs_steps: ${n_obs_steps} + n_action_steps: ${n_action_steps} + fps: 10 + past_action: ${past_action_visible} + n_envs: null + +dataset: + _target_: diffusion_policy.dataset.robot_image_dataset.RobotImageDataset + zarr_path: data/useless.zarr + batch_size: ${dataloader.batch_size} + horizon: ${horizon} + pad_before: ${eval:'${n_obs_steps}-1'} + pad_after: ${eval:'${n_action_steps}-1'} + seed: 42 + val_ratio: 0.02 + max_train_episodes: null diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/config/task/default_task_16.yaml b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/config/task/default_task_16.yaml new file mode 100644 index 00000000..6dd619a0 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/config/task/default_task_16.yaml @@ -0,0 +1,50 @@ +name: task_config + +image_shape: &image_shape [3, -1, -1] +shape_meta: &shape_meta + # acceptable types: rgb, low_dim + obs: + head_cam: + shape: *image_shape + type: rgb + # front_cam: + # shape: *image_shape + # type: rgb + # left_cam: + # shape: *image_shape + # type: rgb + # right_cam: + # shape: *image_shape + # type: rgb + agent_pos: + shape: [16] + type: low_dim + action: + shape: [16] + +env_runner: + _target_: diffusion_policy.env_runner.pusht_image_runner.PushTImageRunner + n_train: 6 + n_train_vis: 2 + train_start_seed: 0 + n_test: 50 + n_test_vis: 4 + legacy_test: True + test_start_seed: 100000 + max_steps: 300 + n_obs_steps: ${n_obs_steps} + n_action_steps: ${n_action_steps} + fps: 10 + past_action: ${past_action_visible} + n_envs: null + +dataset: + _target_: diffusion_policy.dataset.robot_image_dataset.RobotImageDataset + zarr_path: data/useless.zarr + batch_size: ${dataloader.batch_size} + horizon: ${horizon} + pad_before: ${eval:'${n_obs_steps}-1'} + pad_after: ${eval:'${n_action_steps}-1'} + seed: 42 + val_ratio: 0.02 + max_train_episodes: null diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/dataset/base_dataset.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/dataset/base_dataset.py new file mode 100644 index 00000000..4ec33860 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/dataset/base_dataset.py @@ -0,0 +1,54 @@ +from typing import Dict + +import torch +import torch.nn +from diffusion_policy.model.common.normalizer import LinearNormalizer + + +class BaseLowdimDataset(torch.utils.data.Dataset): + + def get_validation_dataset(self) -> "BaseLowdimDataset": + # return an empty dataset by default + return BaseLowdimDataset() + + def get_normalizer(self, **kwargs) -> LinearNormalizer: + raise NotImplementedError() + + def get_all_actions(self) -> torch.Tensor: + raise NotImplementedError() + + def __len__(self) -> int: + return 0 + + def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]: + """ + output: + obs: T, Do + action: T, Da + """ + raise NotImplementedError() + + +class BaseImageDataset(torch.utils.data.Dataset): + + def get_validation_dataset(self) -> "BaseLowdimDataset": + # return an empty dataset by default + return BaseImageDataset() + + def get_normalizer(self, **kwargs) -> LinearNormalizer: + raise NotImplementedError() + + def get_all_actions(self) -> torch.Tensor: + raise NotImplementedError() + + def __len__(self) -> int: + return 0 + + def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]: + """ + output: + obs: + key: T, * + action: T, Da + """ + raise NotImplementedError() diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/dataset/robot_image_dataset.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/dataset/robot_image_dataset.py new file mode 100644 index 00000000..d935680f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/dataset/robot_image_dataset.py @@ -0,0 +1,185 @@ +from typing import Dict +import numba +import torch +import numpy as np +import copy +from diffusion_policy.common.pytorch_util import dict_apply +from diffusion_policy.common.replay_buffer import ReplayBuffer +from diffusion_policy.common.sampler import ( + SequenceSampler, + get_val_mask, + downsample_mask, +) +from diffusion_policy.model.common.normalizer import LinearNormalizer +from diffusion_policy.dataset.base_dataset import BaseImageDataset +from diffusion_policy.common.normalize_util import get_image_range_normalizer +import pdb + + +class RobotImageDataset(BaseImageDataset): + + def __init__( + self, + zarr_path, + horizon=1, + pad_before=0, + pad_after=0, + seed=42, + val_ratio=0.0, + batch_size=128, + max_train_episodes=None, + ): + + super().__init__() + self.replay_buffer = ReplayBuffer.copy_from_path( + zarr_path, + # keys=['head_camera', 'front_camera', 'left_camera', 'right_camera', 'state', 'action'], + keys=["head_camera", "state", "action"], + ) + + val_mask = get_val_mask(n_episodes=self.replay_buffer.n_episodes, val_ratio=val_ratio, seed=seed) + train_mask = ~val_mask + train_mask = downsample_mask(mask=train_mask, max_n=max_train_episodes, seed=seed) + + self.sampler = SequenceSampler( + replay_buffer=self.replay_buffer, + sequence_length=horizon, + pad_before=pad_before, + pad_after=pad_after, + episode_mask=train_mask, + ) + self.train_mask = train_mask + self.horizon = horizon + self.pad_before = pad_before + self.pad_after = pad_after + + self.batch_size = batch_size + sequence_length = self.sampler.sequence_length + self.buffers = { + k: np.zeros((batch_size, sequence_length, *v.shape[1:]), dtype=v.dtype) + for k, v in self.sampler.replay_buffer.items() + } + self.buffers_torch = {k: torch.from_numpy(v) for k, v in self.buffers.items()} + for v in self.buffers_torch.values(): + v.pin_memory() + + def get_validation_dataset(self): + val_set = copy.copy(self) + val_set.sampler = SequenceSampler( + replay_buffer=self.replay_buffer, + sequence_length=self.horizon, + pad_before=self.pad_before, + pad_after=self.pad_after, + episode_mask=~self.train_mask, + ) + val_set.train_mask = ~self.train_mask + return val_set + + def get_normalizer(self, mode="limits", **kwargs): + data = { + "action": self.replay_buffer["action"], + "agent_pos": self.replay_buffer["state"], + } + normalizer = LinearNormalizer() + normalizer.fit(data=data, last_n_dims=1, mode=mode, **kwargs) + normalizer["head_cam"] = get_image_range_normalizer() + normalizer["front_cam"] = get_image_range_normalizer() + normalizer["left_cam"] = get_image_range_normalizer() + normalizer["right_cam"] = get_image_range_normalizer() + return normalizer + + def __len__(self) -> int: + return len(self.sampler) + + def _sample_to_data(self, sample): + agent_pos = sample["state"].astype(np.float32) # (agent_posx2, block_posex3) + head_cam = np.moveaxis(sample["head_camera"], -1, 1) / 255 + # front_cam = np.moveaxis(sample['front_camera'],-1,1)/255 + # left_cam = np.moveaxis(sample['left_camera'],-1,1)/255 + # right_cam = np.moveaxis(sample['right_camera'],-1,1)/255 + + data = { + "obs": { + "head_cam": head_cam, # T, 3, H, W + # 'front_cam': front_cam, # T, 3, H, W + # 'left_cam': left_cam, # T, 3, H, W + # 'right_cam': right_cam, # T, 3, H, W + "agent_pos": agent_pos, # T, D + }, + "action": sample["action"].astype(np.float32), # T, D + } + return data + + def __getitem__(self, idx) -> Dict[str, torch.Tensor]: + if isinstance(idx, slice): + raise NotImplementedError # Specialized + elif isinstance(idx, int): + sample = self.sampler.sample_sequence(idx) + sample = dict_apply(sample, torch.from_numpy) + return sample + elif isinstance(idx, np.ndarray): + assert len(idx) == self.batch_size + for k, v in self.sampler.replay_buffer.items(): + batch_sample_sequence( + self.buffers[k], + v, + self.sampler.indices, + idx, + self.sampler.sequence_length, + ) + return self.buffers_torch + else: + raise ValueError(idx) + + def postprocess(self, samples, device): + agent_pos = samples["state"].to(device, non_blocking=True) + head_cam = samples["head_camera"].to(device, non_blocking=True) / 255.0 + # front_cam = samples['front_camera'].to(device, non_blocking=True) / 255.0 + # left_cam = samples['left_camera'].to(device, non_blocking=True) / 255.0 + # right_cam = samples['right_camera'].to(device, non_blocking=True) / 255.0 + action = samples["action"].to(device, non_blocking=True) + return { + "obs": { + "head_cam": head_cam, # B, T, 3, H, W + # 'front_cam': front_cam, # B, T, 3, H, W + # 'left_cam': left_cam, # B, T, 3, H, W + # 'right_cam': right_cam, # B, T, 3, H, W + "agent_pos": agent_pos, # B, T, D + }, + "action": action, # B, T, D + } + + +def _batch_sample_sequence( + data: np.ndarray, + input_arr: np.ndarray, + indices: np.ndarray, + idx: np.ndarray, + sequence_length: int, +): + for i in numba.prange(len(idx)): + buffer_start_idx, buffer_end_idx, sample_start_idx, sample_end_idx = indices[idx[i]] + data[i, sample_start_idx:sample_end_idx] = input_arr[buffer_start_idx:buffer_end_idx] + if sample_start_idx > 0: + data[i, :sample_start_idx] = data[i, sample_start_idx] + if sample_end_idx < sequence_length: + data[i, sample_end_idx:] = data[i, sample_end_idx - 1] + + +_batch_sample_sequence_sequential = numba.jit(_batch_sample_sequence, nopython=True, parallel=False) +_batch_sample_sequence_parallel = numba.jit(_batch_sample_sequence, nopython=True, parallel=True) + + +def batch_sample_sequence( + data: np.ndarray, + input_arr: np.ndarray, + indices: np.ndarray, + idx: np.ndarray, + sequence_length: int, +): + batch_size = len(idx) + assert data.shape == (batch_size, sequence_length, *input_arr.shape[1:]) + if batch_size >= 16 and data.nbytes // batch_size >= 2**16: + _batch_sample_sequence_parallel(data, input_arr, indices, idx, sequence_length) + else: + _batch_sample_sequence_sequential(data, input_arr, indices, idx, sequence_length) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/env_runner/dp_runner.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/env_runner/dp_runner.py new file mode 100644 index 00000000..96ecd02e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/env_runner/dp_runner.py @@ -0,0 +1,102 @@ +import torch +import os +import numpy as np +import hydra +from pathlib import Path +from collections import deque + +import yaml +from datetime import datetime +import importlib +import dill +from argparse import ArgumentParser +from diffusion_policy.common.pytorch_util import dict_apply +from diffusion_policy.policy.base_image_policy import BaseImagePolicy + + +class DPRunner: + + def __init__( + self, + eval_episodes=20, + max_steps=300, + n_obs_steps=3, + n_action_steps=8, + fps=10, + crf=22, + tqdm_interval_sec=5.0, + task_name=None, + ): + self.task_name = task_name + self.eval_episodes = eval_episodes + self.fps = fps + self.crf = crf + self.n_obs_steps = n_obs_steps + self.n_action_steps = n_action_steps + self.max_steps = max_steps + self.tqdm_interval_sec = tqdm_interval_sec + + self.obs = deque(maxlen=n_obs_steps + 1) + self.env = None + + def stack_last_n_obs(self, all_obs, n_steps): + assert len(all_obs) > 0 + all_obs = list(all_obs) + if isinstance(all_obs[0], np.ndarray): + result = np.zeros((n_steps, ) + all_obs[-1].shape, dtype=all_obs[-1].dtype) + start_idx = -min(n_steps, len(all_obs)) + result[start_idx:] = np.array(all_obs[start_idx:]) + if n_steps > len(all_obs): + # pad + result[:start_idx] = result[start_idx] + elif isinstance(all_obs[0], torch.Tensor): + result = torch.zeros((n_steps, ) + all_obs[-1].shape, dtype=all_obs[-1].dtype) + start_idx = -min(n_steps, len(all_obs)) + result[start_idx:] = torch.stack(all_obs[start_idx:]) + if n_steps > len(all_obs): + # pad + result[:start_idx] = result[start_idx] + else: + raise RuntimeError(f"Unsupported obs type {type(all_obs[0])}") + return result + + def reset_obs(self): + self.obs.clear() + + def update_obs(self, current_obs): + self.obs.append(current_obs) + + def get_n_steps_obs(self): + assert len(self.obs) > 0, "no observation is recorded, please update obs first" + + result = dict() + for key in self.obs[0].keys(): + result[key] = self.stack_last_n_obs([obs[key] for obs in self.obs], self.n_obs_steps) + + return result + + def get_action(self, policy: BaseImagePolicy, observaton=None): + device, dtype = policy.device, policy.dtype + if observaton is not None: + self.obs.append(observaton) # update + obs = self.get_n_steps_obs() + + # create obs dict + np_obs_dict = dict(obs) + # device transfer + obs_dict = dict_apply(np_obs_dict, lambda x: torch.from_numpy(x).to(device=device)) + # run policy + with torch.no_grad(): + obs_dict_input = {} # flush unused keys + obs_dict_input["head_cam"] = obs_dict["head_cam"].unsqueeze(0) + # obs_dict_input['front_cam'] = obs_dict['front_cam'].unsqueeze(0) + obs_dict_input["left_cam"] = obs_dict["left_cam"].unsqueeze(0) + obs_dict_input["right_cam"] = obs_dict["right_cam"].unsqueeze(0) + obs_dict_input["agent_pos"] = obs_dict["agent_pos"].unsqueeze(0) + + action_dict = policy.predict_action(obs_dict_input) + + # device_transfer + np_action_dict = dict_apply(action_dict, lambda x: x.detach().to("cpu").numpy()) + action = np_action_dict["action"].squeeze(0)[:self.n_action_steps] + return action diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/action_ae/__init__.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/action_ae/__init__.py new file mode 100644 index 00000000..de289fd6 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/action_ae/__init__.py @@ -0,0 +1,64 @@ +import torch +import torch.nn as nn +from torch.utils.data import DataLoader +import abc + +from typing import Optional, Union + +import diffusion_policy.model.bet.utils as utils + + +class AbstractActionAE(utils.SaveModule, abc.ABC): + + @abc.abstractmethod + def fit_model( + self, + input_dataloader: DataLoader, + eval_dataloader: DataLoader, + obs_encoding_net: Optional[nn.Module] = None, + ) -> None: + pass + + @abc.abstractmethod + def encode_into_latent( + self, + input_action: torch.Tensor, + input_rep: Optional[torch.Tensor], + ) -> torch.Tensor: + """ + Given the input action, discretize it. + + Inputs: + input_action (shape: ... x action_dim): The input action to discretize. This can be in a batch, + and is generally assumed that the last dimnesion is the action dimension. + + Outputs: + discretized_action (shape: ... x num_tokens): The discretized action. + """ + raise NotImplementedError + + @abc.abstractmethod + def decode_actions( + self, + latent_action_batch: Optional[torch.Tensor], + input_rep_batch: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """ + Given a discretized action, convert it to a continuous action. + + Inputs: + latent_action_batch (shape: ... x num_tokens): The discretized action + generated by the discretizer. + + Outputs: + continuous_action (shape: ... x action_dim): The continuous action. + """ + raise NotImplementedError + + @property + @abc.abstractmethod + def num_latents(self) -> Union[int, float]: + """ + Number of possible latents for this generator, useful for state priors that use softmax. + """ + return float("inf") diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/action_ae/discretizers/k_means.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/action_ae/discretizers/k_means.py new file mode 100644 index 00000000..b4e17839 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/action_ae/discretizers/k_means.py @@ -0,0 +1,136 @@ +import torch +import numpy as np + +import tqdm + +from typing import Optional, Tuple, Union +from diffusion_policy.model.common.dict_of_tensor_mixin import DictOfTensorMixin + + +class KMeansDiscretizer(DictOfTensorMixin): + """ + Simplified and modified version of KMeans algorithm from sklearn. + """ + + def __init__( + self, + action_dim: int, + num_bins: int = 100, + predict_offsets: bool = False, + ): + super().__init__() + self.n_bins = num_bins + self.action_dim = action_dim + self.predict_offsets = predict_offsets + + def fit_discretizer(self, input_actions: torch.Tensor) -> None: + assert (self.action_dim == input_actions.shape[-1] + ), f"Input action dimension {self.action_dim} does not match fitted model {input_actions.shape[-1]}" + + flattened_actions = input_actions.view(-1, self.action_dim) + cluster_centers = KMeansDiscretizer._kmeans(flattened_actions, ncluster=self.n_bins) + self.params_dict["bin_centers"] = cluster_centers + + @property + def suggested_actions(self) -> torch.Tensor: + return self.params_dict["bin_centers"] + + @classmethod + def _kmeans(cls, x: torch.Tensor, ncluster: int = 512, niter: int = 50): + """ + Simple k-means clustering algorithm adapted from Karpathy's minGPT library + https://github.com/karpathy/minGPT/blob/master/play_image.ipynb + """ + N, D = x.size() + c = x[torch.randperm(N)[:ncluster]] # init clusters at random + + pbar = tqdm.trange(niter) + pbar.set_description("K-means clustering") + for i in pbar: + # assign all pixels to the closest codebook element + a = ((x[:, None, :] - c[None, :, :])**2).sum(-1).argmin(1) + # move each codebook element to be the mean of the pixels that assigned to it + c = torch.stack([x[a == k].mean(0) for k in range(ncluster)]) + # re-assign any poorly positioned codebook elements + nanix = torch.any(torch.isnan(c), dim=1) + ndead = nanix.sum().item() + if ndead: + tqdm.tqdm.write("done step %d/%d, re-initialized %d dead clusters" % (i + 1, niter, ndead)) + c[nanix] = x[torch.randperm(N)[:ndead]] # re-init dead clusters + return c + + def encode_into_latent(self, input_action: torch.Tensor, input_rep: Optional[torch.Tensor] = None) -> torch.Tensor: + """ + Given the input action, discretize it using the k-Means clustering algorithm. + + Inputs: + input_action (shape: ... x action_dim): The input action to discretize. This can be in a batch, + and is generally assumed that the last dimnesion is the action dimension. + + Outputs: + discretized_action (shape: ... x num_tokens): The discretized action. + If self.predict_offsets is True, then the offsets are also returned. + """ + assert (input_action.shape[-1] == self.action_dim), "Input action dimension does not match fitted model" + + # flatten the input action + flattened_actions = input_action.view(-1, self.action_dim) + + # get the closest cluster center + closest_cluster_center = torch.argmin( + torch.sum( + (flattened_actions[:, None, :] - self.params_dict["bin_centers"][None, :, :])**2, + dim=2, + ), + dim=1, + ) + # Reshape to the original shape + discretized_action = closest_cluster_center.view(input_action.shape[:-1] + (1, )) + + if self.predict_offsets: + # decode from latent and get the difference + reconstructed_action = self.decode_actions(discretized_action) + offsets = input_action - reconstructed_action + return (discretized_action, offsets) + else: + # return the one-hot vector + return discretized_action + + def decode_actions( + self, + latent_action_batch: torch.Tensor, + input_rep_batch: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """ + Given the latent action, reconstruct the original action. + + Inputs: + latent_action (shape: ... x 1): The latent action to reconstruct. This can be in a batch, + and is generally assumed that the last dimension is the action dimension. If the latent_action_batch + is a tuple, then it is assumed to be (discretized_action, offsets). + + Outputs: + reconstructed_action (shape: ... x action_dim): The reconstructed action. + """ + offsets = None + if type(latent_action_batch) == tuple: + latent_action_batch, offsets = latent_action_batch + # get the closest cluster center + closest_cluster_center = self.params_dict["bin_centers"][latent_action_batch] + # Reshape to the original shape + reconstructed_action = closest_cluster_center.view(latent_action_batch.shape[:-1] + (self.action_dim, )) + if offsets is not None: + reconstructed_action += offsets + return reconstructed_action + + @property + def discretized_space(self) -> int: + return self.n_bins + + @property + def latent_dim(self) -> int: + return 1 + + @property + def num_latents(self) -> int: + return self.n_bins diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/latent_generators/latent_generator.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/latent_generators/latent_generator.py new file mode 100644 index 00000000..f0e38b2d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/latent_generators/latent_generator.py @@ -0,0 +1,67 @@ +import abc +import torch +from typing import Tuple, Optional + +import diffusion_policy.model.bet.utils as utils + + +class AbstractLatentGenerator(abc.ABC, utils.SaveModule): + """ + Abstract class for a generative model that can generate latents given observation representations. + + In the probabilisitc sense, this model fits and samples from P(latent|observation) given some observation. + """ + + @abc.abstractmethod + def get_latent_and_loss( + self, + obs_rep: torch.Tensor, + target_latents: torch.Tensor, + seq_masks: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Given a set of observation representation and generated latents, get the encoded latent and the loss. + + Inputs: + input_action: Batch of the actions taken in the multimodal demonstrations. + target_latents: Batch of the latents that the generator should learn to generate the actions from. + seq_masks: Batch of masks that indicate which timesteps are valid. + + Outputs: + latent: The sampled latent from the observation. + loss: The loss of the latent generator. + """ + pass + + @abc.abstractmethod + def generate_latents(self, seq_obses: torch.Tensor, seq_masks: torch.Tensor) -> torch.Tensor: + """ + Given a batch of sequences of observations, generate a batch of sequences of latents. + + Inputs: + seq_obses: Batch of sequences of observations, of shape seq x batch x dim, following the transformer convention. + seq_masks: Batch of sequences of masks, of shape seq x batch, following the transformer convention. + + Outputs: + seq_latents: Batch of sequences of latents of shape seq x batch x latent_dim. + """ + pass + + def get_optimizer(self, weight_decay: float, learning_rate: float, betas: Tuple[float, + float]) -> torch.optim.Optimizer: + """ + Default optimizer class. Override this if you want to use a different optimizer. + """ + return torch.optim.Adam(self.parameters(), lr=learning_rate, weight_decay=weight_decay, betas=betas) + + +class LatentGeneratorDataParallel(torch.nn.DataParallel): + + def get_latent_and_loss(self, *args, **kwargs): + return self.module.get_latent_and_loss(*args, **kwargs) # type: ignore + + def generate_latents(self, *args, **kwargs): + return self.module.generate_latents(*args, **kwargs) # type: ignore + + def get_optimizer(self, *args, **kwargs): + return self.module.get_optimizer(*args, **kwargs) # type: ignore diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/latent_generators/mingpt.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/latent_generators/mingpt.py new file mode 100644 index 00000000..c67b0ac7 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/latent_generators/mingpt.py @@ -0,0 +1,177 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import einops +import diffusion_policy.model.bet.latent_generators.latent_generator as latent_generator + +import diffusion_policy.model.bet.libraries.mingpt.model as mingpt_model +import diffusion_policy.model.bet.libraries.mingpt.trainer as mingpt_trainer +from diffusion_policy.model.bet.libraries.loss_fn import FocalLoss, soft_cross_entropy + +from typing import Optional, Tuple + + +class MinGPT(latent_generator.AbstractLatentGenerator): + + def __init__( + self, + input_dim: int, + n_layer: int = 12, + n_head: int = 12, + n_embd: int = 768, + embd_pdrop: float = 0.1, + resid_pdrop: float = 0.1, + attn_pdrop: float = 0.1, + block_size: int = 128, + vocab_size: int = 50257, + latent_dim: int = 768, # Ignore, used for compatibility with other models. + action_dim: int = 0, + discrete_input: bool = False, + predict_offsets: bool = False, + offset_loss_scale: float = 1.0, + focal_loss_gamma: float = 0.0, + **kwargs): + super().__init__() + self.input_size = input_dim + self.n_layer = n_layer + self.n_head = n_head + self.n_embd = n_embd + self.embd_pdrop = embd_pdrop + self.resid_pdrop = resid_pdrop + self.attn_pdrop = attn_pdrop + self.block_size = block_size + self.vocab_size = vocab_size + self.action_dim = action_dim + self.predict_offsets = predict_offsets + self.offset_loss_scale = offset_loss_scale + self.focal_loss_gamma = focal_loss_gamma + for k, v in kwargs.items(): + setattr(self, k, v) + + gpt_config = mingpt_model.GPTConfig( + input_size=self.input_size, + vocab_size=(self.vocab_size * (1 + self.action_dim) if self.predict_offsets else self.vocab_size), + block_size=self.block_size, + n_layer=n_layer, + n_head=n_head, + n_embd=n_embd, + discrete_input=discrete_input, + embd_pdrop=embd_pdrop, + resid_pdrop=resid_pdrop, + attn_pdrop=attn_pdrop, + ) + + self.model = mingpt_model.GPT(gpt_config) + + def get_latent_and_loss( + self, + obs_rep: torch.Tensor, + target_latents: torch.Tensor, + seq_masks: Optional[torch.Tensor] = None, + return_loss_components: bool = False, + ) -> Tuple[torch.Tensor, torch.Tensor]: + # Unlike torch.transformers, GPT takes in batch x seq_len x embd_dim + # obs_rep = einops.rearrange(obs_rep, "seq batch embed -> batch seq embed") + # target_latents = einops.rearrange( + # target_latents, "seq batch embed -> batch seq embed" + # ) + # While this has been trained autoregressively, + # there is no reason why it needs to be so. + # We can just use the observation as the input and the next latent as the target. + if self.predict_offsets: + target_latents, target_offsets = target_latents + is_soft_target = (target_latents.shape[-1] == self.vocab_size) and (self.vocab_size != 1) + if is_soft_target: + target_latents = target_latents.view(-1, target_latents.size(-1)) + criterion = soft_cross_entropy + else: + target_latents = target_latents.view(-1) + if self.vocab_size == 1: + # unify k-means (target_class == 0) and GMM (target_prob == 1) + target_latents = torch.zeros_like(target_latents) + criterion = FocalLoss(gamma=self.focal_loss_gamma) + if self.predict_offsets: + output, _ = self.model(obs_rep) + logits = output[:, :, :self.vocab_size] + offsets = output[:, :, self.vocab_size:] + batch = logits.shape[0] + seq = logits.shape[1] + offsets = einops.rearrange( + offsets, + "N T (V A) -> (N T) V A", # N = batch, T = seq + V=self.vocab_size, + A=self.action_dim, + ) + # calculate (optionally soft) cross entropy and offset losses + class_loss = criterion(logits.view(-1, logits.size(-1)), target_latents) + # offset loss is only calculated on the target class + # if soft targets, argmax is considered the target class + selected_offsets = offsets[ + torch.arange(offsets.size(0)), + (target_latents.argmax(dim=-1).view(-1) if is_soft_target else target_latents.view(-1)), + ] + offset_loss = self.offset_loss_scale * F.mse_loss(selected_offsets, target_offsets.view( + -1, self.action_dim)) + loss = offset_loss + class_loss + logits = einops.rearrange(logits, "batch seq classes -> seq batch classes") + offsets = einops.rearrange( + offsets, + "(N T) V A -> T N V A", # ? N, T order? Anyway does not affect loss and training (might affect visualization) + N=batch, + T=seq, + ) + if return_loss_components: + return ( + (logits, offsets), + loss, + { + "offset": offset_loss, + "class": class_loss, + "total": loss + }, + ) + else: + return (logits, offsets), loss + else: + logits, _ = self.model(obs_rep) + loss = criterion(logits.view(-1, logits.size(-1)), target_latents) + logits = einops.rearrange( + logits, "batch seq classes -> seq batch classes" + ) # ? N, T order? Anyway does not affect loss and training (might affect visualization) + if return_loss_components: + return logits, loss, {"class": loss, "total": loss} + else: + return logits, loss + + def generate_latents(self, obs_rep: torch.Tensor) -> torch.Tensor: + batch, seq, embed = obs_rep.shape + + output, _ = self.model(obs_rep, None) + if self.predict_offsets: + logits = output[:, :, :self.vocab_size] + offsets = output[:, :, self.vocab_size:] + offsets = einops.rearrange( + offsets, + "N T (V A) -> (N T) V A", # N = batch, T = seq + V=self.vocab_size, + A=self.action_dim, + ) + else: + logits = output + probs = F.softmax(logits, dim=-1) + batch, seq, choices = probs.shape + # Sample from the multinomial distribution, one per row. + sampled_data = torch.multinomial(probs.view(-1, choices), num_samples=1) + sampled_data = einops.rearrange(sampled_data, "(batch seq) 1 -> batch seq 1", batch=batch, seq=seq) + if self.predict_offsets: + sampled_offsets = offsets[torch.arange(offsets.shape[0]), + sampled_data.flatten()].view(batch, seq, self.action_dim) + + return (sampled_data, sampled_offsets) + else: + return sampled_data + + def get_optimizer(self, weight_decay: float, learning_rate: float, betas: Tuple[float, + float]) -> torch.optim.Optimizer: + trainer_cfg = mingpt_trainer.TrainerConfig(weight_decay=weight_decay, learning_rate=learning_rate, betas=betas) + return self.model.configure_optimizers(trainer_cfg) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/latent_generators/transformer.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/latent_generators/transformer.py new file mode 100644 index 00000000..d46f4093 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/latent_generators/transformer.py @@ -0,0 +1,99 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import einops +import diffusion_policy.model.bet.latent_generators.latent_generator as latent_generator + +from diffusion_policy.model.diffusion.transformer_for_diffusion import ( + TransformerForDiffusion, ) +from diffusion_policy.model.bet.libraries.loss_fn import FocalLoss, soft_cross_entropy + +from typing import Optional, Tuple + + +class Transformer(latent_generator.AbstractLatentGenerator): + + def __init__(self, input_dim: int, num_bins: int, action_dim: int, horizon: int, focal_loss_gamma: float, + offset_loss_scale: float, **kwargs): + super().__init__() + self.model = TransformerForDiffusion(input_dim=input_dim, + output_dim=num_bins * (1 + action_dim), + horizon=horizon, + **kwargs) + self.vocab_size = num_bins + self.focal_loss_gamma = focal_loss_gamma + self.offset_loss_scale = offset_loss_scale + self.action_dim = action_dim + + def get_optimizer(self, **kwargs) -> torch.optim.Optimizer: + return self.model.configure_optimizers(**kwargs) + + def get_latent_and_loss( + self, + obs_rep: torch.Tensor, + target_latents: torch.Tensor, + return_loss_components=True, + ) -> Tuple[torch.Tensor, torch.Tensor]: + target_latents, target_offsets = target_latents + target_latents = target_latents.view(-1) + criterion = FocalLoss(gamma=self.focal_loss_gamma) + + t = torch.tensor(0, device=self.model.device) + output = self.model(obs_rep, t) + logits = output[:, :, :self.vocab_size] + offsets = output[:, :, self.vocab_size:] + batch = logits.shape[0] + seq = logits.shape[1] + offsets = einops.rearrange( + offsets, + "N T (V A) -> (N T) V A", # N = batch, T = seq + V=self.vocab_size, + A=self.action_dim, + ) + # calculate (optionally soft) cross entropy and offset losses + class_loss = criterion(logits.view(-1, logits.size(-1)), target_latents) + # offset loss is only calculated on the target class + # if soft targets, argmax is considered the target class + selected_offsets = offsets[ + torch.arange(offsets.size(0)), + target_latents.view(-1), + ] + offset_loss = self.offset_loss_scale * F.mse_loss(selected_offsets, target_offsets.view(-1, self.action_dim)) + loss = offset_loss + class_loss + logits = einops.rearrange(logits, "batch seq classes -> seq batch classes") + offsets = einops.rearrange( + offsets, + "(N T) V A -> T N V A", # ? N, T order? Anyway does not affect loss and training (might affect visualization) + N=batch, + T=seq, + ) + return ( + (logits, offsets), + loss, + { + "offset": offset_loss, + "class": class_loss, + "total": loss + }, + ) + + def generate_latents(self, obs_rep: torch.Tensor) -> torch.Tensor: + t = torch.tensor(0, device=self.model.device) + output = self.model(obs_rep, t) + logits = output[:, :, :self.vocab_size] + offsets = output[:, :, self.vocab_size:] + offsets = einops.rearrange( + offsets, + "N T (V A) -> (N T) V A", # N = batch, T = seq + V=self.vocab_size, + A=self.action_dim, + ) + + probs = F.softmax(logits, dim=-1) + batch, seq, choices = probs.shape + # Sample from the multinomial distribution, one per row. + sampled_data = torch.multinomial(probs.view(-1, choices), num_samples=1) + sampled_data = einops.rearrange(sampled_data, "(batch seq) 1 -> batch seq 1", batch=batch, seq=seq) + sampled_offsets = offsets[torch.arange(offsets.shape[0]), + sampled_data.flatten()].view(batch, seq, self.action_dim) + return (sampled_data, sampled_offsets) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/libraries/loss_fn.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/libraries/loss_fn.py new file mode 100644 index 00000000..47817dda --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/libraries/loss_fn.py @@ -0,0 +1,165 @@ +from typing import Optional, Sequence + +import torch +from torch import Tensor +from torch import nn +from torch.nn import functional as F + + +# Reference: https://github.com/pytorch/pytorch/issues/11959 +def soft_cross_entropy( + input: torch.Tensor, + target: torch.Tensor, +) -> torch.Tensor: + """ + Args: + input: (batch_size, num_classes): tensor of raw logits + target: (batch_size, num_classes): tensor of class probability; sum(target) == 1 + + Returns: + loss: (batch_size,) + """ + log_probs = torch.log_softmax(input, dim=-1) + # target is a distribution + loss = F.kl_div(log_probs, target, reduction="batchmean") + return loss + + +# Focal loss implementation +# Source: https://github.com/AdeelH/pytorch-multi-class-focal-loss/blob/master/focal_loss.py +# MIT License +# +# Copyright (c) 2020 Adeel Hassan +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +class FocalLoss(nn.Module): + """Focal Loss, as described in https://arxiv.org/abs/1708.02002. + It is essentially an enhancement to cross entropy loss and is + useful for classification tasks when there is a large class imbalance. + x is expected to contain raw, unnormalized scores for each class. + y is expected to contain class labels. + Shape: + - x: (batch_size, C) or (batch_size, C, d1, d2, ..., dK), K > 0. + - y: (batch_size,) or (batch_size, d1, d2, ..., dK), K > 0. + """ + + def __init__( + self, + alpha: Optional[Tensor] = None, + gamma: float = 0.0, + reduction: str = "mean", + ignore_index: int = -100, + ): + """Constructor. + Args: + alpha (Tensor, optional): Weights for each class. Defaults to None. + gamma (float, optional): A constant, as described in the paper. + Defaults to 0. + reduction (str, optional): 'mean', 'sum' or 'none'. + Defaults to 'mean'. + ignore_index (int, optional): class label to ignore. + Defaults to -100. + """ + if reduction not in ("mean", "sum", "none"): + raise ValueError('Reduction must be one of: "mean", "sum", "none".') + + super().__init__() + self.alpha = alpha + self.gamma = gamma + self.ignore_index = ignore_index + self.reduction = reduction + + self.nll_loss = nn.NLLLoss(weight=alpha, reduction="none", ignore_index=ignore_index) + + def __repr__(self): + arg_keys = ["alpha", "gamma", "ignore_index", "reduction"] + arg_vals = [self.__dict__[k] for k in arg_keys] + arg_strs = [f"{k}={v}" for k, v in zip(arg_keys, arg_vals)] + arg_str = ", ".join(arg_strs) + return f"{type(self).__name__}({arg_str})" + + def forward(self, x: Tensor, y: Tensor) -> Tensor: + if x.ndim > 2: + # (N, C, d1, d2, ..., dK) --> (N * d1 * ... * dK, C) + c = x.shape[1] + x = x.permute(0, *range(2, x.ndim), 1).reshape(-1, c) + # (N, d1, d2, ..., dK) --> (N * d1 * ... * dK,) + y = y.view(-1) + + unignored_mask = y != self.ignore_index + y = y[unignored_mask] + if len(y) == 0: + return 0.0 + x = x[unignored_mask] + + # compute weighted cross entropy term: -alpha * log(pt) + # (alpha is already part of self.nll_loss) + log_p = F.log_softmax(x, dim=-1) + ce = self.nll_loss(log_p, y) + + # get true class column from each row + all_rows = torch.arange(len(x)) + log_pt = log_p[all_rows, y] + + # compute focal term: (1 - pt)^gamma + pt = log_pt.exp() + focal_term = (1 - pt)**self.gamma + + # the full loss: -alpha * ((1 - pt)^gamma) * log(pt) + loss = focal_term * ce + + if self.reduction == "mean": + loss = loss.mean() + elif self.reduction == "sum": + loss = loss.sum() + + return loss + + +def focal_loss( + alpha: Optional[Sequence] = None, + gamma: float = 0.0, + reduction: str = "mean", + ignore_index: int = -100, + device="cpu", + dtype=torch.float32, +) -> FocalLoss: + """Factory function for FocalLoss. + Args: + alpha (Sequence, optional): Weights for each class. Will be converted + to a Tensor if not None. Defaults to None. + gamma (float, optional): A constant, as described in the paper. + Defaults to 0. + reduction (str, optional): 'mean', 'sum' or 'none'. + Defaults to 'mean'. + ignore_index (int, optional): class label to ignore. + Defaults to -100. + device (str, optional): Device to move alpha to. Defaults to 'cpu'. + dtype (torch.dtype, optional): dtype to cast alpha to. + Defaults to torch.float32. + Returns: + A FocalLoss object + """ + if alpha is not None: + if not isinstance(alpha, Tensor): + alpha = torch.tensor(alpha) + alpha = alpha.to(device=device, dtype=dtype) + + fl = FocalLoss(alpha=alpha, gamma=gamma, reduction=reduction, ignore_index=ignore_index) + return fl diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/libraries/mingpt/LICENSE b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/libraries/mingpt/LICENSE new file mode 100644 index 00000000..87bfb153 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/libraries/mingpt/LICENSE @@ -0,0 +1,8 @@ +The MIT License (MIT) Copyright (c) 2020 Andrej Karpathy + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/libraries/mingpt/__init__.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/libraries/mingpt/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/libraries/mingpt/model.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/libraries/mingpt/model.py new file mode 100644 index 00000000..ab9e1c48 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/libraries/mingpt/model.py @@ -0,0 +1,231 @@ +""" +GPT model: +- the initial stem consists of a combination of token encoding and a positional encoding +- the meat of it is a uniform sequence of Transformer blocks + - each Transformer is a sequential combination of a 1-hidden-layer MLP block and a self-attention block + - all blocks feed into a central residual pathway similar to resnets +- the final decoder is a linear projection into a vanilla Softmax classifier +""" + +import math +import logging + +import torch +import torch.nn as nn +from torch.nn import functional as F + +logger = logging.getLogger(__name__) + + +class GPTConfig: + """base GPT config, params common to all GPT versions""" + + embd_pdrop = 0.1 + resid_pdrop = 0.1 + attn_pdrop = 0.1 + discrete_input = False + input_size = 10 + n_embd = 768 + n_layer = 12 + + def __init__(self, vocab_size, block_size, **kwargs): + self.vocab_size = vocab_size + self.block_size = block_size + for k, v in kwargs.items(): + setattr(self, k, v) + + +class GPT1Config(GPTConfig): + """GPT-1 like network roughly 125M params""" + + n_layer = 12 + n_head = 12 + n_embd = 768 + + +class CausalSelfAttention(nn.Module): + """ + A vanilla multi-head masked self-attention layer with a projection at the end. + It is possible to use torch.nn.MultiheadAttention here but I am including an + explicit implementation here to show that there is nothing too scary here. + """ + + def __init__(self, config): + super().__init__() + assert config.n_embd % config.n_head == 0 + # key, query, value projections for all heads + self.key = nn.Linear(config.n_embd, config.n_embd) + self.query = nn.Linear(config.n_embd, config.n_embd) + self.value = nn.Linear(config.n_embd, config.n_embd) + # regularization + self.attn_drop = nn.Dropout(config.attn_pdrop) + self.resid_drop = nn.Dropout(config.resid_pdrop) + # output projection + self.proj = nn.Linear(config.n_embd, config.n_embd) + # causal mask to ensure that attention is only applied to the left in the input sequence + self.register_buffer( + "mask", + torch.tril(torch.ones(config.block_size, config.block_size)).view(1, 1, config.block_size, + config.block_size), + ) + self.n_head = config.n_head + + def forward(self, x): + ( + B, + T, + C, + ) = x.size() # batch size, sequence length, embedding dimensionality (n_embd) + + # calculate query, key, values for all heads in batch and move head forward to be the batch dim + k = (self.key(x).view(B, T, self.n_head, C // self.n_head).transpose(1, 2)) # (B, nh, T, hs) + q = (self.query(x).view(B, T, self.n_head, C // self.n_head).transpose(1, 2)) # (B, nh, T, hs) + v = (self.value(x).view(B, T, self.n_head, C // self.n_head).transpose(1, 2)) # (B, nh, T, hs) + + # causal self-attention; Self-attend: (B, nh, T, hs) x (B, nh, hs, T) -> (B, nh, T, T) + att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(k.size(-1))) + att = att.masked_fill(self.mask[:, :, :T, :T] == 0, float("-inf")) + att = F.softmax(att, dim=-1) + att = self.attn_drop(att) + y = att @ v # (B, nh, T, T) x (B, nh, T, hs) -> (B, nh, T, hs) + y = (y.transpose(1, 2).contiguous().view(B, T, C)) # re-assemble all head outputs side by side + + # output projection + y = self.resid_drop(self.proj(y)) + return y + + +class Block(nn.Module): + """an unassuming Transformer block""" + + def __init__(self, config): + super().__init__() + self.ln1 = nn.LayerNorm(config.n_embd) + self.ln2 = nn.LayerNorm(config.n_embd) + self.attn = CausalSelfAttention(config) + self.mlp = nn.Sequential( + nn.Linear(config.n_embd, 4 * config.n_embd), + nn.GELU(), + nn.Linear(4 * config.n_embd, config.n_embd), + nn.Dropout(config.resid_pdrop), + ) + + def forward(self, x): + x = x + self.attn(self.ln1(x)) + x = x + self.mlp(self.ln2(x)) + return x + + +class GPT(nn.Module): + """the full GPT language model, with a context size of block_size""" + + def __init__(self, config: GPTConfig): + super().__init__() + + # input embedding stem + if config.discrete_input: + self.tok_emb = nn.Embedding(config.vocab_size, config.n_embd) + else: + self.tok_emb = nn.Linear(config.input_size, config.n_embd) + self.discrete_input = config.discrete_input + self.pos_emb = nn.Parameter(torch.zeros(1, config.block_size, config.n_embd)) + self.drop = nn.Dropout(config.embd_pdrop) + # transformer + self.blocks = nn.Sequential(*[Block(config) for _ in range(config.n_layer)]) + # decoder head + self.ln_f = nn.LayerNorm(config.n_embd) + self.head = nn.Linear(config.n_embd, config.vocab_size, bias=False) + + self.block_size = config.block_size + self.apply(self._init_weights) + + logger.info("number of parameters: %e", sum(p.numel() for p in self.parameters())) + + def get_block_size(self): + return self.block_size + + def _init_weights(self, module): + if isinstance(module, (nn.Linear, nn.Embedding)): + torch.nn.init.normal_(module.weight, mean=0.0, std=0.02) + if isinstance(module, nn.Linear) and module.bias is not None: + torch.nn.init.zeros_(module.bias) + elif isinstance(module, nn.LayerNorm): + torch.nn.init.zeros_(module.bias) + torch.nn.init.ones_(module.weight) + elif isinstance(module, GPT): + torch.nn.init.normal_(module.pos_emb, mean=0.0, std=0.02) + + def configure_optimizers(self, train_config): + """ + This long function is unfortunately doing something very simple and is being very defensive: + We are separating out all parameters of the model into two buckets: those that will experience + weight decay for regularization and those that won't (biases, and layernorm/embedding weights). + We are then returning the PyTorch optimizer object. + """ + + # separate out all parameters to those that will and won't experience regularizing weight decay + decay = set() + no_decay = set() + whitelist_weight_modules = (torch.nn.Linear, ) + blacklist_weight_modules = (torch.nn.LayerNorm, torch.nn.Embedding) + for mn, m in self.named_modules(): + for pn, p in m.named_parameters(): + fpn = "%s.%s" % (mn, pn) if mn else pn # full param name + + if pn.endswith("bias"): + # all biases will not be decayed + no_decay.add(fpn) + elif pn.endswith("weight") and isinstance(m, whitelist_weight_modules): + # weights of whitelist modules will be weight decayed + decay.add(fpn) + elif pn.endswith("weight") and isinstance(m, blacklist_weight_modules): + # weights of blacklist modules will NOT be weight decayed + no_decay.add(fpn) + + # special case the position embedding parameter in the root GPT module as not decayed + no_decay.add("pos_emb") + + # validate that we considered every parameter + param_dict = {pn: p for pn, p in self.named_parameters()} + inter_params = decay & no_decay + union_params = decay | no_decay + assert (len(inter_params) == 0), "parameters %s made it into both decay/no_decay sets!" % (str(inter_params), ) + assert (len(param_dict.keys() - + union_params) == 0), "parameters %s were not separated into either decay/no_decay set!" % ( + str(param_dict.keys() - union_params), ) + + # create the pytorch optimizer object + optim_groups = [ + { + "params": [param_dict[pn] for pn in sorted(list(decay))], + "weight_decay": train_config.weight_decay, + }, + { + "params": [param_dict[pn] for pn in sorted(list(no_decay))], + "weight_decay": 0.0, + }, + ] + optimizer = torch.optim.AdamW(optim_groups, lr=train_config.learning_rate, betas=train_config.betas) + return optimizer + + def forward(self, idx, targets=None): + if self.discrete_input: + b, t = idx.size() + else: + b, t, dim = idx.size() + assert t <= self.block_size, "Cannot forward, model block size is exhausted." + + # forward the GPT model + token_embeddings = self.tok_emb(idx) # each index maps to a (learnable) vector + position_embeddings = self.pos_emb[:, :t, :] # each position maps to a (learnable) vector + x = self.drop(token_embeddings + position_embeddings) + x = self.blocks(x) + x = self.ln_f(x) + logits = self.head(x) + + # if we are given some desired targets also calculate the loss + loss = None + if targets is not None: + loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1)) + + return logits, loss diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/libraries/mingpt/trainer.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/libraries/mingpt/trainer.py new file mode 100644 index 00000000..10fd3c45 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/libraries/mingpt/trainer.py @@ -0,0 +1,145 @@ +""" +Simple training loop; Boilerplate that could apply to any arbitrary neural network, +so nothing in this file really has anything to do with GPT specifically. +""" + +import math +import logging + +from tqdm import tqdm +import numpy as np + +import torch +import torch.optim as optim +from torch.optim.lr_scheduler import LambdaLR +from torch.utils.data.dataloader import DataLoader + +logger = logging.getLogger(__name__) + + +class TrainerConfig: + # optimization parameters + max_epochs = 10 + batch_size = 64 + learning_rate = 3e-4 + betas = (0.9, 0.95) + grad_norm_clip = 1.0 + weight_decay = 0.1 # only applied on matmul weights + # learning rate decay params: linear warmup followed by cosine decay to 10% of original + lr_decay = False + warmup_tokens = 375e6 # these two numbers come from the GPT-3 paper, but may not be good defaults elsewhere + final_tokens = 260e9 # (at what point we reach 10% of original LR) + # checkpoint settings + ckpt_path = None + num_workers = 0 # for DataLoader + + def __init__(self, **kwargs): + for k, v in kwargs.items(): + setattr(self, k, v) + + +class Trainer: + + def __init__(self, model, train_dataset, test_dataset, config): + self.model = model + self.train_dataset = train_dataset + self.test_dataset = test_dataset + self.config = config + + # take over whatever gpus are on the system + self.device = "cpu" + if torch.cuda.is_available(): + self.device = torch.cuda.current_device() + self.model = torch.nn.DataParallel(self.model).to(self.device) + + def save_checkpoint(self): + # DataParallel wrappers keep raw model object in .module attribute + raw_model = self.model.module if hasattr(self.model, "module") else self.model + logger.info("saving %s", self.config.ckpt_path) + torch.save(raw_model.state_dict(), self.config.ckpt_path) + + def train(self): + model, config = self.model, self.config + raw_model = model.module if hasattr(self.model, "module") else model + optimizer = raw_model.configure_optimizers(config) + + def run_epoch(loader, is_train): + model.train(is_train) + + losses = [] + pbar = (tqdm(enumerate(loader), total=len(loader)) if is_train else enumerate(loader)) + for it, (x, y) in pbar: + + # place data on the correct device + x = x.to(self.device) + y = y.to(self.device) + + # forward the model + with torch.set_grad_enabled(is_train): + logits, loss = model(x, y) + loss = (loss.mean()) # collapse all losses if they are scattered on multiple gpus + losses.append(loss.item()) + + if is_train: + + # backprop and update the parameters + model.zero_grad() + loss.backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), config.grad_norm_clip) + optimizer.step() + + # decay the learning rate based on our progress + if config.lr_decay: + self.tokens += (y >= 0).sum() # number of tokens processed this step (i.e. label is not -100) + if self.tokens < config.warmup_tokens: + # linear warmup + lr_mult = float(self.tokens) / float(max(1, config.warmup_tokens)) + else: + # cosine learning rate decay + progress = float(self.tokens - config.warmup_tokens) / float( + max(1, config.final_tokens - config.warmup_tokens)) + lr_mult = max(0.1, 0.5 * (1.0 + math.cos(math.pi * progress))) + lr = config.learning_rate * lr_mult + for param_group in optimizer.param_groups: + param_group["lr"] = lr + else: + lr = config.learning_rate + + # report progress + pbar.set_description( # type: ignore + f"epoch {epoch+1} iter {it}: train loss {loss.item():.5f}. lr {lr:e}") + + if not is_train: + test_loss = float(np.mean(losses)) + logger.info("test loss: %f", test_loss) + return test_loss + + best_loss = float("inf") + self.tokens = 0 # counter used for learning rate decay + + train_loader = DataLoader( + self.train_dataset, + shuffle=True, + pin_memory=True, + batch_size=config.batch_size, + num_workers=config.num_workers, + ) + if self.test_dataset is not None: + test_loader = DataLoader( + self.test_dataset, + shuffle=True, + pin_memory=True, + batch_size=config.batch_size, + num_workers=config.num_workers, + ) + + for epoch in range(config.max_epochs): + run_epoch(train_loader, is_train=True) + if self.test_dataset is not None: + test_loss = run_epoch(test_loader, is_train=False) + + # supports early stopping based on the test loss, or just save always if no test set is provided + good_model = self.test_dataset is None or test_loss < best_loss + if self.config.ckpt_path is not None and good_model: + best_loss = test_loss + self.save_checkpoint() diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/libraries/mingpt/utils.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/libraries/mingpt/utils.py new file mode 100644 index 00000000..1065ce7b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/libraries/mingpt/utils.py @@ -0,0 +1,49 @@ +import numpy as np +import torch +import torch.nn as nn +from torch.nn import functional as F + + +def set_seed(seed): + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + +def top_k_logits(logits, k): + v, ix = torch.topk(logits, k) + out = logits.clone() + out[out < v[:, [-1]]] = -float("Inf") + return out + + +@torch.no_grad() +def sample(model, x, steps, temperature=1.0, sample=False, top_k=None): + """ + take a conditioning sequence of indices in x (of shape (b,t)) and predict the next token in + the sequence, feeding the predictions back into the model each time. Clearly the sampling + has quadratic complexity unlike an RNN that is only linear, and has a finite context window + of block_size, unlike an RNN that has an infinite context window. + """ + block_size = model.get_block_size() + model.eval() + for k in range(steps): + x_cond = (x if x.size(1) <= block_size else x[:, -block_size:]) # crop context if needed + logits, _ = model(x_cond) + # pluck the logits at the final step and scale by temperature + logits = logits[:, -1, :] / temperature + # optionally crop probabilities to only the top k options + if top_k is not None: + logits = top_k_logits(logits, top_k) + # apply softmax to convert to probabilities + probs = F.softmax(logits, dim=-1) + # sample from the distribution or take the most likely + if sample: + ix = torch.multinomial(probs, num_samples=1) + else: + _, ix = torch.topk(probs, k=1, dim=-1) + # append to the sequence and continue + x = torch.cat((x, ix), dim=1) + + return x diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/utils.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/utils.py new file mode 100644 index 00000000..3b75b732 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/bet/utils.py @@ -0,0 +1,130 @@ +import os + +from collections import OrderedDict +from typing import List, Optional + +import einops +import numpy as np +import torch +import torch.nn as nn + +from torch.utils.data import random_split +import wandb + + +def mlp(input_dim, hidden_dim, output_dim, hidden_depth, output_mod=None): + if hidden_depth == 0: + mods = [nn.Linear(input_dim, output_dim)] + else: + mods = [nn.Linear(input_dim, hidden_dim), nn.ReLU(inplace=True)] + for i in range(hidden_depth - 1): + mods += [nn.Linear(hidden_dim, hidden_dim), nn.ReLU(inplace=True)] + mods.append(nn.Linear(hidden_dim, output_dim)) + if output_mod is not None: + mods.append(output_mod) + trunk = nn.Sequential(*mods) + return trunk + + +class eval_mode: + + def __init__(self, *models, no_grad=False): + self.models = models + self.no_grad = no_grad + self.no_grad_context = torch.no_grad() + + def __enter__(self): + self.prev_states = [] + for model in self.models: + self.prev_states.append(model.training) + model.train(False) + if self.no_grad: + self.no_grad_context.__enter__() + + def __exit__(self, *args): + if self.no_grad: + self.no_grad_context.__exit__(*args) + for model, state in zip(self.models, self.prev_states): + model.train(state) + return False + + +def freeze_module(module: nn.Module) -> nn.Module: + for param in module.parameters(): + param.requires_grad = False + module.eval() + return module + + +def set_seed_everywhere(seed): + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + np.random.seed(seed) + random.seed(seed) + + +def shuffle_along_axis(a, axis): + idx = np.random.rand(*a.shape).argsort(axis=axis) + return np.take_along_axis(a, idx, axis=axis) + + +def transpose_batch_timestep(*args): + return (einops.rearrange(arg, "b t ... -> t b ...") for arg in args) + + +class TrainWithLogger: + + def reset_log(self): + self.log_components = OrderedDict() + + def log_append(self, log_key, length, loss_components): + for key, value in loss_components.items(): + key_name = f"{log_key}/{key}" + count, sum = self.log_components.get(key_name, (0, 0.0)) + self.log_components[key_name] = ( + count + length, + sum + (length * value.detach().cpu().item()), + ) + + def flush_log(self, epoch, iterator=None): + log_components = OrderedDict() + iterator_log_component = OrderedDict() + for key, value in self.log_components.items(): + count, sum = value + to_log = sum / count + log_components[key] = to_log + # Set the iterator status + log_key, name_key = key.split("/") + iterator_log_name = f"{log_key[0]}{name_key[0]}".upper() + iterator_log_component[iterator_log_name] = to_log + postfix = ",".join("{}:{:.2e}".format(key, iterator_log_component[key]) + for key in iterator_log_component.keys()) + if iterator is not None: + iterator.set_postfix_str(postfix) + wandb.log(log_components, step=epoch) + self.log_components = OrderedDict() + + +class SaveModule(nn.Module): + + def set_snapshot_path(self, path): + self.snapshot_path = path + print(f"Setting snapshot path to {self.snapshot_path}") + + def save_snapshot(self): + os.makedirs(self.snapshot_path, exist_ok=True) + torch.save(self.state_dict(), self.snapshot_path / "snapshot.pth") + + def load_snapshot(self): + self.load_state_dict(torch.load(self.snapshot_path / "snapshot.pth")) + + +def split_datasets(dataset, train_fraction=0.95, random_seed=42): + dataset_length = len(dataset) + lengths = [ + int(train_fraction * dataset_length), + dataset_length - int(train_fraction * dataset_length), + ] + train_set, val_set = random_split(dataset, lengths, generator=torch.Generator().manual_seed(random_seed)) + return train_set, val_set diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/common/dict_of_tensor_mixin.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/common/dict_of_tensor_mixin.py new file mode 100644 index 00000000..358da9fe --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/common/dict_of_tensor_mixin.py @@ -0,0 +1,50 @@ +import torch +import torch.nn as nn + + +class DictOfTensorMixin(nn.Module): + + def __init__(self, params_dict=None): + super().__init__() + if params_dict is None: + params_dict = nn.ParameterDict() + self.params_dict = params_dict + + @property + def device(self): + return next(iter(self.parameters())).device + + def _load_from_state_dict( + self, + state_dict, + prefix, + local_metadata, + strict, + missing_keys, + unexpected_keys, + error_msgs, + ): + + def dfs_add(dest, keys, value: torch.Tensor): + if len(keys) == 1: + dest[keys[0]] = value + return + + if keys[0] not in dest: + dest[keys[0]] = nn.ParameterDict() + dfs_add(dest[keys[0]], keys[1:], value) + + def load_dict(state_dict, prefix): + out_dict = nn.ParameterDict() + for key, value in state_dict.items(): + value: torch.Tensor + if key.startswith(prefix): + param_keys = key[len(prefix):].split(".")[1:] + # if len(param_keys) == 0: + # import pdb; pdb.set_trace() + dfs_add(out_dict, param_keys, value.clone()) + return out_dict + + self.params_dict = load_dict(state_dict, prefix + "params_dict") + self.params_dict.requires_grad_(False) + return diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/common/lr_scheduler.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/common/lr_scheduler.py new file mode 100644 index 00000000..c97f3052 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/common/lr_scheduler.py @@ -0,0 +1,55 @@ +from diffusers.optimization import ( + Union, + SchedulerType, + Optional, + Optimizer, + TYPE_TO_SCHEDULER_FUNCTION, +) + + +def get_scheduler( + name: Union[str, SchedulerType], + optimizer: Optimizer, + num_warmup_steps: Optional[int] = None, + num_training_steps: Optional[int] = None, + **kwargs, +): + """ + Added kwargs vs diffuser's original implementation + + Unified API to get any scheduler from its name. + + Args: + name (`str` or `SchedulerType`): + The name of the scheduler to use. + optimizer (`torch.optim.Optimizer`): + The optimizer that will be used during training. + num_warmup_steps (`int`, *optional*): + The number of warmup steps to do. This is not required by all schedulers (hence the argument being + optional), the function will raise an error if it's unset and the scheduler type requires it. + num_training_steps (`int``, *optional*): + The number of training steps to do. This is not required by all schedulers (hence the argument being + optional), the function will raise an error if it's unset and the scheduler type requires it. + """ + name = SchedulerType(name) + schedule_func = TYPE_TO_SCHEDULER_FUNCTION[name] + if name == SchedulerType.CONSTANT: + return schedule_func(optimizer, **kwargs) + + # All other schedulers require `num_warmup_steps` + if num_warmup_steps is None: + raise ValueError(f"{name} requires `num_warmup_steps`, please provide that argument.") + + if name == SchedulerType.CONSTANT_WITH_WARMUP: + return schedule_func(optimizer, num_warmup_steps=num_warmup_steps, **kwargs) + + # All other schedulers require `num_training_steps` + if num_training_steps is None: + raise ValueError(f"{name} requires `num_training_steps`, please provide that argument.") + + return schedule_func( + optimizer, + num_warmup_steps=num_warmup_steps, + num_training_steps=num_training_steps, + **kwargs, + ) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/common/module_attr_mixin.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/common/module_attr_mixin.py new file mode 100644 index 00000000..e33efe29 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/common/module_attr_mixin.py @@ -0,0 +1,16 @@ +import torch.nn as nn + + +class ModuleAttrMixin(nn.Module): + + def __init__(self): + super().__init__() + self._dummy_variable = nn.Parameter() + + @property + def device(self): + return next(iter(self.parameters())).device + + @property + def dtype(self): + return next(iter(self.parameters())).dtype diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/common/normalizer.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/common/normalizer.py new file mode 100644 index 00000000..c8d362ca --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/common/normalizer.py @@ -0,0 +1,369 @@ +from typing import Union, Dict + +import unittest +import zarr +import numpy as np +import torch +import torch.nn as nn +from diffusion_policy.common.pytorch_util import dict_apply +from diffusion_policy.model.common.dict_of_tensor_mixin import DictOfTensorMixin + + +class LinearNormalizer(DictOfTensorMixin): + avaliable_modes = ["limits", "gaussian"] + + @torch.no_grad() + def fit( + self, + data: Union[Dict, torch.Tensor, np.ndarray, zarr.Array], + last_n_dims=1, + dtype=torch.float32, + mode="limits", + output_max=1.0, + output_min=-1.0, + range_eps=1e-4, + fit_offset=True, + ): + if isinstance(data, dict): + for key, value in data.items(): + self.params_dict[key] = _fit( + value, + last_n_dims=last_n_dims, + dtype=dtype, + mode=mode, + output_max=output_max, + output_min=output_min, + range_eps=range_eps, + fit_offset=fit_offset, + ) + else: + self.params_dict["_default"] = _fit( + data, + last_n_dims=last_n_dims, + dtype=dtype, + mode=mode, + output_max=output_max, + output_min=output_min, + range_eps=range_eps, + fit_offset=fit_offset, + ) + + def __call__(self, x: Union[Dict, torch.Tensor, np.ndarray]) -> torch.Tensor: + return self.normalize(x) + + def __getitem__(self, key: str): + return SingleFieldLinearNormalizer(self.params_dict[key]) + + def __setitem__(self, key: str, value: "SingleFieldLinearNormalizer"): + self.params_dict[key] = value.params_dict + + def _normalize_impl(self, x, forward=True): + if isinstance(x, dict): + result = dict() + for key, value in x.items(): + params = self.params_dict[key] + result[key] = _normalize(value, params, forward=forward) + return result + else: + if "_default" not in self.params_dict: + raise RuntimeError("Not initialized") + params = self.params_dict["_default"] + return _normalize(x, params, forward=forward) + + def normalize(self, x: Union[Dict, torch.Tensor, np.ndarray]) -> torch.Tensor: + return self._normalize_impl(x, forward=True) + + def unnormalize(self, x: Union[Dict, torch.Tensor, np.ndarray]) -> torch.Tensor: + return self._normalize_impl(x, forward=False) + + def get_input_stats(self) -> Dict: + if len(self.params_dict) == 0: + raise RuntimeError("Not initialized") + if len(self.params_dict) == 1 and "_default" in self.params_dict: + return self.params_dict["_default"]["input_stats"] + + result = dict() + for key, value in self.params_dict.items(): + if key != "_default": + result[key] = value["input_stats"] + return result + + def get_output_stats(self, key="_default"): + input_stats = self.get_input_stats() + if "min" in input_stats: + # no dict + return dict_apply(input_stats, self.normalize) + + result = dict() + for key, group in input_stats.items(): + this_dict = dict() + for name, value in group.items(): + this_dict[name] = self.normalize({key: value})[key] + result[key] = this_dict + return result + + +class SingleFieldLinearNormalizer(DictOfTensorMixin): + avaliable_modes = ["limits", "gaussian"] + + @torch.no_grad() + def fit( + self, + data: Union[torch.Tensor, np.ndarray, zarr.Array], + last_n_dims=1, + dtype=torch.float32, + mode="limits", + output_max=1.0, + output_min=-1.0, + range_eps=1e-4, + fit_offset=True, + ): + self.params_dict = _fit( + data, + last_n_dims=last_n_dims, + dtype=dtype, + mode=mode, + output_max=output_max, + output_min=output_min, + range_eps=range_eps, + fit_offset=fit_offset, + ) + + @classmethod + def create_fit(cls, data: Union[torch.Tensor, np.ndarray, zarr.Array], **kwargs): + obj = cls() + obj.fit(data, **kwargs) + return obj + + @classmethod + def create_manual( + cls, + scale: Union[torch.Tensor, np.ndarray], + offset: Union[torch.Tensor, np.ndarray], + input_stats_dict: Dict[str, Union[torch.Tensor, np.ndarray]], + ): + + def to_tensor(x): + if not isinstance(x, torch.Tensor): + x = torch.from_numpy(x) + x = x.flatten() + return x + + # check + for x in [offset] + list(input_stats_dict.values()): + assert x.shape == scale.shape + assert x.dtype == scale.dtype + + params_dict = nn.ParameterDict({ + "scale": to_tensor(scale), + "offset": to_tensor(offset), + "input_stats": nn.ParameterDict(dict_apply(input_stats_dict, to_tensor)), + }) + return cls(params_dict) + + @classmethod + def create_identity(cls, dtype=torch.float32): + scale = torch.tensor([1], dtype=dtype) + offset = torch.tensor([0], dtype=dtype) + input_stats_dict = { + "min": torch.tensor([-1], dtype=dtype), + "max": torch.tensor([1], dtype=dtype), + "mean": torch.tensor([0], dtype=dtype), + "std": torch.tensor([1], dtype=dtype), + } + return cls.create_manual(scale, offset, input_stats_dict) + + def normalize(self, x: Union[torch.Tensor, np.ndarray]) -> torch.Tensor: + return _normalize(x, self.params_dict, forward=True) + + def unnormalize(self, x: Union[torch.Tensor, np.ndarray]) -> torch.Tensor: + return _normalize(x, self.params_dict, forward=False) + + def get_input_stats(self): + return self.params_dict["input_stats"] + + def get_output_stats(self): + return dict_apply(self.params_dict["input_stats"], self.normalize) + + def __call__(self, x: Union[torch.Tensor, np.ndarray]) -> torch.Tensor: + return self.normalize(x) + + +def _fit( + data: Union[torch.Tensor, np.ndarray, zarr.Array], + last_n_dims=1, + dtype=torch.float32, + mode="limits", + output_max=1.0, + output_min=-1.0, + range_eps=1e-4, + fit_offset=True, +): + assert mode in ["limits", "gaussian"] + assert last_n_dims >= 0 + assert output_max > output_min + + # convert data to torch and type + if isinstance(data, zarr.Array): + data = data[:] + if isinstance(data, np.ndarray): + data = torch.from_numpy(data) + if dtype is not None: + data = data.type(dtype) + + # convert shape + dim = 1 + if last_n_dims > 0: + dim = np.prod(data.shape[-last_n_dims:]) + data = data.reshape(-1, dim) + + # compute input stats min max mean std + input_min, _ = data.min(axis=0) + input_max, _ = data.max(axis=0) + input_mean = data.mean(axis=0) + input_std = data.std(axis=0) + + # compute scale and offset + if mode == "limits": + if fit_offset: + # unit scale + input_range = input_max - input_min + ignore_dim = input_range < range_eps + input_range[ignore_dim] = output_max - output_min + scale = (output_max - output_min) / input_range + offset = output_min - scale * input_min + offset[ignore_dim] = (output_max + output_min) / 2 - input_min[ignore_dim] + # ignore dims scaled to mean of output max and min + else: + # use this when data is pre-zero-centered. + assert output_max > 0 + assert output_min < 0 + # unit abs + output_abs = min(abs(output_min), abs(output_max)) + input_abs = torch.maximum(torch.abs(input_min), torch.abs(input_max)) + ignore_dim = input_abs < range_eps + input_abs[ignore_dim] = output_abs + # don't scale constant channels + scale = output_abs / input_abs + offset = torch.zeros_like(input_mean) + elif mode == "gaussian": + ignore_dim = input_std < range_eps + scale = input_std.clone() + scale[ignore_dim] = 1 + scale = 1 / scale + + if fit_offset: + offset = -input_mean * scale + else: + offset = torch.zeros_like(input_mean) + + # save + this_params = nn.ParameterDict({ + "scale": + scale, + "offset": + offset, + "input_stats": + nn.ParameterDict({ + "min": input_min, + "max": input_max, + "mean": input_mean, + "std": input_std, + }), + }) + for p in this_params.parameters(): + p.requires_grad_(False) + return this_params + + +def _normalize(x, params, forward=True): + assert "scale" in params + if isinstance(x, np.ndarray): + x = torch.from_numpy(x) + scale = params["scale"] + offset = params["offset"] + x = x.to(device=scale.device, dtype=scale.dtype) + src_shape = x.shape + # import pdb + # pdb.set_trace() + x = x.reshape(-1, scale.shape[0]) + if forward: + x = x * scale + offset + else: + x = (x - offset) / scale + x = x.reshape(src_shape) + return x + + +def test(): + data = torch.zeros((100, 10, 9, 2)).uniform_() + data[..., 0, 0] = 0 + + normalizer = SingleFieldLinearNormalizer() + normalizer.fit(data, mode="limits", last_n_dims=2) + datan = normalizer.normalize(data) + assert datan.shape == data.shape + assert np.allclose(datan.max(), 1.0) + assert np.allclose(datan.min(), -1.0) + dataun = normalizer.unnormalize(datan) + assert torch.allclose(data, dataun, atol=1e-7) + + input_stats = normalizer.get_input_stats() + output_stats = normalizer.get_output_stats() + + normalizer = SingleFieldLinearNormalizer() + normalizer.fit(data, mode="limits", last_n_dims=1, fit_offset=False) + datan = normalizer.normalize(data) + assert datan.shape == data.shape + assert np.allclose(datan.max(), 1.0, atol=1e-3) + assert np.allclose(datan.min(), 0.0, atol=1e-3) + dataun = normalizer.unnormalize(datan) + assert torch.allclose(data, dataun, atol=1e-7) + + data = torch.zeros((100, 10, 9, 2)).uniform_() + normalizer = SingleFieldLinearNormalizer() + normalizer.fit(data, mode="gaussian", last_n_dims=0) + datan = normalizer.normalize(data) + assert datan.shape == data.shape + assert np.allclose(datan.mean(), 0.0, atol=1e-3) + assert np.allclose(datan.std(), 1.0, atol=1e-3) + dataun = normalizer.unnormalize(datan) + assert torch.allclose(data, dataun, atol=1e-7) + + # dict + data = torch.zeros((100, 10, 9, 2)).uniform_() + data[..., 0, 0] = 0 + + normalizer = LinearNormalizer() + normalizer.fit(data, mode="limits", last_n_dims=2) + datan = normalizer.normalize(data) + assert datan.shape == data.shape + assert np.allclose(datan.max(), 1.0) + assert np.allclose(datan.min(), -1.0) + dataun = normalizer.unnormalize(datan) + assert torch.allclose(data, dataun, atol=1e-7) + + input_stats = normalizer.get_input_stats() + output_stats = normalizer.get_output_stats() + + data = { + "obs": torch.zeros((1000, 128, 9, 2)).uniform_() * 512, + "action": torch.zeros((1000, 128, 2)).uniform_() * 512, + } + normalizer = LinearNormalizer() + normalizer.fit(data) + datan = normalizer.normalize(data) + dataun = normalizer.unnormalize(datan) + for key in data: + assert torch.allclose(data[key], dataun[key], atol=1e-4) + + input_stats = normalizer.get_input_stats() + output_stats = normalizer.get_output_stats() + + state_dict = normalizer.state_dict() + n = LinearNormalizer() + n.load_state_dict(state_dict) + datan = n.normalize(data) + dataun = n.unnormalize(datan) + for key in data: + assert torch.allclose(data[key], dataun[key], atol=1e-4) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/common/rotation_transformer.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/common/rotation_transformer.py new file mode 100644 index 00000000..faf182a0 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/common/rotation_transformer.py @@ -0,0 +1,97 @@ +from typing import Union +import pytorch3d.transforms as pt +import torch +import numpy as np +import functools + + +class RotationTransformer: + valid_reps = ["axis_angle", "euler_angles", "quaternion", "rotation_6d", "matrix"] + + def __init__( + self, + from_rep="axis_angle", + to_rep="rotation_6d", + from_convention=None, + to_convention=None, + ): + """ + Valid representations + + Always use matrix as intermediate representation. + """ + assert from_rep != to_rep + assert from_rep in self.valid_reps + assert to_rep in self.valid_reps + if from_rep == "euler_angles": + assert from_convention is not None + if to_rep == "euler_angles": + assert to_convention is not None + + forward_funcs = list() + inverse_funcs = list() + + if from_rep != "matrix": + funcs = [ + getattr(pt, f"{from_rep}_to_matrix"), + getattr(pt, f"matrix_to_{from_rep}"), + ] + if from_convention is not None: + funcs = [functools.partial(func, convention=from_convention) for func in funcs] + forward_funcs.append(funcs[0]) + inverse_funcs.append(funcs[1]) + + if to_rep != "matrix": + funcs = [ + getattr(pt, f"matrix_to_{to_rep}"), + getattr(pt, f"{to_rep}_to_matrix"), + ] + if to_convention is not None: + funcs = [functools.partial(func, convention=to_convention) for func in funcs] + forward_funcs.append(funcs[0]) + inverse_funcs.append(funcs[1]) + + inverse_funcs = inverse_funcs[::-1] + + self.forward_funcs = forward_funcs + self.inverse_funcs = inverse_funcs + + @staticmethod + def _apply_funcs(x: Union[np.ndarray, torch.Tensor], funcs: list) -> Union[np.ndarray, torch.Tensor]: + x_ = x + if isinstance(x, np.ndarray): + x_ = torch.from_numpy(x) + x_: torch.Tensor + for func in funcs: + x_ = func(x_) + y = x_ + if isinstance(x, np.ndarray): + y = x_.numpy() + return y + + def forward(self, x: Union[np.ndarray, torch.Tensor]) -> Union[np.ndarray, torch.Tensor]: + return self._apply_funcs(x, self.forward_funcs) + + def inverse(self, x: Union[np.ndarray, torch.Tensor]) -> Union[np.ndarray, torch.Tensor]: + return self._apply_funcs(x, self.inverse_funcs) + + +def test(): + tf = RotationTransformer() + + rotvec = np.random.uniform(-2 * np.pi, 2 * np.pi, size=(1000, 3)) + rot6d = tf.forward(rotvec) + new_rotvec = tf.inverse(rot6d) + + from scipy.spatial.transform import Rotation + + diff = Rotation.from_rotvec(rotvec) * Rotation.from_rotvec(new_rotvec).inv() + dist = diff.magnitude() + assert dist.max() < 1e-7 + + tf = RotationTransformer("rotation_6d", "matrix") + rot6d_wrong = rot6d + np.random.normal(scale=0.1, size=rot6d.shape) + mat = tf.forward(rot6d_wrong) + mat_det = np.linalg.det(mat) + assert np.allclose(mat_det, 1) + # rotaiton_6d will be normalized to rotation matrix diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/common/shape_util.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/common/shape_util.py new file mode 100644 index 00000000..2445d8ad --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/common/shape_util.py @@ -0,0 +1,22 @@ +from typing import Dict, List, Tuple, Callable +import torch +import torch.nn as nn + + +def get_module_device(m: nn.Module): + device = torch.device("cpu") + try: + param = next(iter(m.parameters())) + device = param.device + except StopIteration: + pass + return device + + +@torch.no_grad() +def get_output_shape(input_shape: Tuple[int], net: Callable[[torch.Tensor], torch.Tensor]): + device = get_module_device(net) + test_input = torch.zeros((1, ) + tuple(input_shape), device=device) + test_output = net(test_input) + output_shape = tuple(test_output.shape[1:]) + return output_shape diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/common/tensor_util.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/common/tensor_util.py new file mode 100644 index 00000000..f0fc7dd1 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/common/tensor_util.py @@ -0,0 +1,972 @@ +""" +A collection of utilities for working with nested tensor structures consisting +of numpy arrays and torch tensors. +""" + +import collections +import numpy as np +import torch + + +def recursive_dict_list_tuple_apply(x, type_func_dict): + """ + Recursively apply functions to a nested dictionary or list or tuple, given a dictionary of + {data_type: function_to_apply}. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + type_func_dict (dict): a mapping from data types to the functions to be + applied for each data type. + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + assert list not in type_func_dict + assert tuple not in type_func_dict + assert dict not in type_func_dict + + if isinstance(x, (dict, collections.OrderedDict)): + new_x = (collections.OrderedDict() if isinstance(x, collections.OrderedDict) else dict()) + for k, v in x.items(): + new_x[k] = recursive_dict_list_tuple_apply(v, type_func_dict) + return new_x + elif isinstance(x, (list, tuple)): + ret = [recursive_dict_list_tuple_apply(v, type_func_dict) for v in x] + if isinstance(x, tuple): + ret = tuple(ret) + return ret + else: + for t, f in type_func_dict.items(): + if isinstance(x, t): + return f(x) + else: + raise NotImplementedError("Cannot handle data type %s" % str(type(x))) + + +def map_tensor(x, func): + """ + Apply function @func to torch.Tensor objects in a nested dictionary or + list or tuple. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + func (function): function to apply to each tensor + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: func, + type(None): lambda x: x, + }, + ) + + +def map_ndarray(x, func): + """ + Apply function @func to np.ndarray objects in a nested dictionary or + list or tuple. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + func (function): function to apply to each array + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + np.ndarray: func, + type(None): lambda x: x, + }, + ) + + +def map_tensor_ndarray(x, tensor_func, ndarray_func): + """ + Apply function @tensor_func to torch.Tensor objects and @ndarray_func to + np.ndarray objects in a nested dictionary or list or tuple. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + tensor_func (function): function to apply to each tensor + ndarray_Func (function): function to apply to each array + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: tensor_func, + np.ndarray: ndarray_func, + type(None): lambda x: x, + }, + ) + + +def clone(x): + """ + Clones all torch tensors and numpy arrays in nested dictionary or list + or tuple and returns a new nested structure. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: lambda x: x.clone(), + np.ndarray: lambda x: x.copy(), + type(None): lambda x: x, + }, + ) + + +def detach(x): + """ + Detaches all torch tensors in nested dictionary or list + or tuple and returns a new nested structure. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: lambda x: x.detach(), + }, + ) + + +def to_batch(x): + """ + Introduces a leading batch dimension of 1 for all torch tensors and numpy + arrays in nested dictionary or list or tuple and returns a new nested structure. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: lambda x: x[None, ...], + np.ndarray: lambda x: x[None, ...], + type(None): lambda x: x, + }, + ) + + +def to_sequence(x): + """ + Introduces a time dimension of 1 at dimension 1 for all torch tensors and numpy + arrays in nested dictionary or list or tuple and returns a new nested structure. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: lambda x: x[:, None, ...], + np.ndarray: lambda x: x[:, None, ...], + type(None): lambda x: x, + }, + ) + + +def index_at_time(x, ind): + """ + Indexes all torch tensors and numpy arrays in dimension 1 with index @ind in + nested dictionary or list or tuple and returns a new nested structure. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + ind (int): index + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: lambda x: x[:, ind, ...], + np.ndarray: lambda x: x[:, ind, ...], + type(None): lambda x: x, + }, + ) + + +def unsqueeze(x, dim): + """ + Adds dimension of size 1 at dimension @dim in all torch tensors and numpy arrays + in nested dictionary or list or tuple and returns a new nested structure. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + dim (int): dimension + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: lambda x: x.unsqueeze(dim=dim), + np.ndarray: lambda x: np.expand_dims(x, axis=dim), + type(None): lambda x: x, + }, + ) + + +def contiguous(x): + """ + Makes all torch tensors and numpy arrays contiguous in nested dictionary or + list or tuple and returns a new nested structure. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: lambda x: x.contiguous(), + np.ndarray: lambda x: np.ascontiguousarray(x), + type(None): lambda x: x, + }, + ) + + +def to_device(x, device): + """ + Sends all torch tensors in nested dictionary or list or tuple to device + @device, and returns a new nested structure. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + device (torch.Device): device to send tensors to + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: lambda x, d=device: x.to(d), + type(None): lambda x: x, + }, + ) + + +def to_tensor(x): + """ + Converts all numpy arrays in nested dictionary or list or tuple to + torch tensors (and leaves existing torch Tensors as-is), and returns + a new nested structure. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: lambda x: x, + np.ndarray: lambda x: torch.from_numpy(x), + type(None): lambda x: x, + }, + ) + + +def to_numpy(x): + """ + Converts all torch tensors in nested dictionary or list or tuple to + numpy (and leaves existing numpy arrays as-is), and returns + a new nested structure. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + + def f(tensor): + if tensor.is_cuda: + return tensor.detach().cpu().numpy() + else: + return tensor.detach().numpy() + + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: f, + np.ndarray: lambda x: x, + type(None): lambda x: x, + }, + ) + + +def to_list(x): + """ + Converts all torch tensors and numpy arrays in nested dictionary or list + or tuple to a list, and returns a new nested structure. Useful for + json encoding. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + + def f(tensor): + if tensor.is_cuda: + return tensor.detach().cpu().numpy().tolist() + else: + return tensor.detach().numpy().tolist() + + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: f, + np.ndarray: lambda x: x.tolist(), + type(None): lambda x: x, + }, + ) + + +def to_float(x): + """ + Converts all torch tensors and numpy arrays in nested dictionary or list + or tuple to float type entries, and returns a new nested structure. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: lambda x: x.float(), + np.ndarray: lambda x: x.astype(np.float32), + type(None): lambda x: x, + }, + ) + + +def to_uint8(x): + """ + Converts all torch tensors and numpy arrays in nested dictionary or list + or tuple to uint8 type entries, and returns a new nested structure. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: lambda x: x.byte(), + np.ndarray: lambda x: x.astype(np.uint8), + type(None): lambda x: x, + }, + ) + + +def to_torch(x, device): + """ + Converts all numpy arrays and torch tensors in nested dictionary or list or tuple to + torch tensors on device @device and returns a new nested structure. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + device (torch.Device): device to send tensors to + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return to_device(to_float(to_tensor(x)), device) + + +def to_one_hot_single(tensor, num_class): + """ + Convert tensor to one-hot representation, assuming a certain number of total class labels. + + Args: + tensor (torch.Tensor): tensor containing integer labels + num_class (int): number of classes + + Returns: + x (torch.Tensor): tensor containing one-hot representation of labels + """ + x = torch.zeros(tensor.size() + (num_class, )).to(tensor.device) + x.scatter_(-1, tensor.unsqueeze(-1), 1) + return x + + +def to_one_hot(tensor, num_class): + """ + Convert all tensors in nested dictionary or list or tuple to one-hot representation, + assuming a certain number of total class labels. + + Args: + tensor (dict or list or tuple): a possibly nested dictionary or list or tuple + num_class (int): number of classes + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return map_tensor(tensor, func=lambda x, nc=num_class: to_one_hot_single(x, nc)) + + +def flatten_single(x, begin_axis=1): + """ + Flatten a tensor in all dimensions from @begin_axis onwards. + + Args: + x (torch.Tensor): tensor to flatten + begin_axis (int): which axis to flatten from + + Returns: + y (torch.Tensor): flattened tensor + """ + fixed_size = x.size()[:begin_axis] + _s = list(fixed_size) + [-1] + return x.reshape(*_s) + + +def flatten(x, begin_axis=1): + """ + Flatten all tensors in nested dictionary or list or tuple, from @begin_axis onwards. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + begin_axis (int): which axis to flatten from + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: lambda x, b=begin_axis: flatten_single(x, begin_axis=b), + }, + ) + + +def reshape_dimensions_single(x, begin_axis, end_axis, target_dims): + """ + Reshape selected dimensions in a tensor to a target dimension. + + Args: + x (torch.Tensor): tensor to reshape + begin_axis (int): begin dimension + end_axis (int): end dimension + target_dims (tuple or list): target shape for the range of dimensions + (@begin_axis, @end_axis) + + Returns: + y (torch.Tensor): reshaped tensor + """ + assert begin_axis <= end_axis + assert begin_axis >= 0 + assert end_axis < len(x.shape) + assert isinstance(target_dims, (tuple, list)) + s = x.shape + final_s = [] + for i in range(len(s)): + if i == begin_axis: + final_s.extend(target_dims) + elif i < begin_axis or i > end_axis: + final_s.append(s[i]) + return x.reshape(*final_s) + + +def reshape_dimensions(x, begin_axis, end_axis, target_dims): + """ + Reshape selected dimensions for all tensors in nested dictionary or list or tuple + to a target dimension. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + begin_axis (int): begin dimension + end_axis (int): end dimension + target_dims (tuple or list): target shape for the range of dimensions + (@begin_axis, @end_axis) + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: + lambda x, b=begin_axis, e=end_axis, t=target_dims: reshape_dimensions_single( + x, begin_axis=b, end_axis=e, target_dims=t), + np.ndarray: + lambda x, b=begin_axis, e=end_axis, t=target_dims: reshape_dimensions_single( + x, begin_axis=b, end_axis=e, target_dims=t), + type(None): + lambda x: x, + }, + ) + + +def join_dimensions(x, begin_axis, end_axis): + """ + Joins all dimensions between dimensions (@begin_axis, @end_axis) into a flat dimension, for + all tensors in nested dictionary or list or tuple. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + begin_axis (int): begin dimension + end_axis (int): end dimension + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: + lambda x, b=begin_axis, e=end_axis: reshape_dimensions_single(x, begin_axis=b, end_axis=e, target_dims=[-1] + ), + np.ndarray: + lambda x, b=begin_axis, e=end_axis: reshape_dimensions_single(x, begin_axis=b, end_axis=e, target_dims=[-1] + ), + type(None): + lambda x: x, + }, + ) + + +def expand_at_single(x, size, dim): + """ + Expand a tensor at a single dimension @dim by @size + + Args: + x (torch.Tensor): input tensor + size (int): size to expand + dim (int): dimension to expand + + Returns: + y (torch.Tensor): expanded tensor + """ + assert dim < x.ndimension() + assert x.shape[dim] == 1 + expand_dims = [-1] * x.ndimension() + expand_dims[dim] = size + return x.expand(*expand_dims) + + +def expand_at(x, size, dim): + """ + Expand all tensors in nested dictionary or list or tuple at a single + dimension @dim by @size. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + size (int): size to expand + dim (int): dimension to expand + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return map_tensor(x, lambda t, s=size, d=dim: expand_at_single(t, s, d)) + + +def unsqueeze_expand_at(x, size, dim): + """ + Unsqueeze and expand a tensor at a dimension @dim by @size. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + size (int): size to expand + dim (int): dimension to unsqueeze and expand + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + x = unsqueeze(x, dim) + return expand_at(x, size, dim) + + +def repeat_by_expand_at(x, repeats, dim): + """ + Repeat a dimension by combining expand and reshape operations. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + repeats (int): number of times to repeat the target dimension + dim (int): dimension to repeat on + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + x = unsqueeze_expand_at(x, repeats, dim + 1) + return join_dimensions(x, dim, dim + 1) + + +def named_reduce_single(x, reduction, dim): + """ + Reduce tensor at a dimension by named reduction functions. + + Args: + x (torch.Tensor): tensor to be reduced + reduction (str): one of ["sum", "max", "mean", "flatten"] + dim (int): dimension to be reduced (or begin axis for flatten) + + Returns: + y (torch.Tensor): reduced tensor + """ + assert x.ndimension() > dim + assert reduction in ["sum", "max", "mean", "flatten"] + if reduction == "flatten": + x = flatten(x, begin_axis=dim) + elif reduction == "max": + x = torch.max(x, dim=dim)[0] # [B, D] + elif reduction == "sum": + x = torch.sum(x, dim=dim) + else: + x = torch.mean(x, dim=dim) + return x + + +def named_reduce(x, reduction, dim): + """ + Reduces all tensors in nested dictionary or list or tuple at a dimension + using a named reduction function. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + reduction (str): one of ["sum", "max", "mean", "flatten"] + dim (int): dimension to be reduced (or begin axis for flatten) + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return map_tensor(x, func=lambda t, r=reduction, d=dim: named_reduce_single(t, r, d)) + + +def gather_along_dim_with_dim_single(x, target_dim, source_dim, indices): + """ + This function indexes out a target dimension of a tensor in a structured way, + by allowing a different value to be selected for each member of a flat index + tensor (@indices) corresponding to a source dimension. This can be interpreted + as moving along the source dimension, using the corresponding index value + in @indices to select values for all other dimensions outside of the + source and target dimensions. A common use case is to gather values + in target dimension 1 for each batch member (target dimension 0). + + Args: + x (torch.Tensor): tensor to gather values for + target_dim (int): dimension to gather values along + source_dim (int): dimension to hold constant and use for gathering values + from the other dimensions + indices (torch.Tensor): flat index tensor with same shape as tensor @x along + @source_dim + + Returns: + y (torch.Tensor): gathered tensor, with dimension @target_dim indexed out + """ + assert len(indices.shape) == 1 + assert x.shape[source_dim] == indices.shape[0] + + # unsqueeze in all dimensions except the source dimension + new_shape = [1] * x.ndimension() + new_shape[source_dim] = -1 + indices = indices.reshape(*new_shape) + + # repeat in all dimensions - but preserve shape of source dimension, + # and make sure target_dimension has singleton dimension + expand_shape = list(x.shape) + expand_shape[source_dim] = -1 + expand_shape[target_dim] = 1 + indices = indices.expand(*expand_shape) + + out = x.gather(dim=target_dim, index=indices) + return out.squeeze(target_dim) + + +def gather_along_dim_with_dim(x, target_dim, source_dim, indices): + """ + Apply @gather_along_dim_with_dim_single to all tensors in a nested + dictionary or list or tuple. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + target_dim (int): dimension to gather values along + source_dim (int): dimension to hold constant and use for gathering values + from the other dimensions + indices (torch.Tensor): flat index tensor with same shape as tensor @x along + @source_dim + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return map_tensor( + x, + lambda y, t=target_dim, s=source_dim, i=indices: gather_along_dim_with_dim_single(y, t, s, i), + ) + + +def gather_sequence_single(seq, indices): + """ + Given a tensor with leading dimensions [B, T, ...], gather an element from each sequence in + the batch given an index for each sequence. + + Args: + seq (torch.Tensor): tensor with leading dimensions [B, T, ...] + indices (torch.Tensor): tensor indices of shape [B] + + Return: + y (torch.Tensor): indexed tensor of shape [B, ....] + """ + return gather_along_dim_with_dim_single(seq, target_dim=1, source_dim=0, indices=indices) + + +def gather_sequence(seq, indices): + """ + Given a nested dictionary or list or tuple, gathers an element from each sequence of the batch + for tensors with leading dimensions [B, T, ...]. + + Args: + seq (dict or list or tuple): a possibly nested dictionary or list or tuple with tensors + of leading dimensions [B, T, ...] + indices (torch.Tensor): tensor indices of shape [B] + + Returns: + y (dict or list or tuple): new nested dict-list-tuple with tensors of shape [B, ...] + """ + return gather_along_dim_with_dim(seq, target_dim=1, source_dim=0, indices=indices) + + +def pad_sequence_single(seq, padding, batched=False, pad_same=True, pad_values=None): + """ + Pad input tensor or array @seq in the time dimension (dimension 1). + + Args: + seq (np.ndarray or torch.Tensor): sequence to be padded + padding (tuple): begin and end padding, e.g. [1, 1] pads both begin and end of the sequence by 1 + batched (bool): if sequence has the batch dimension + pad_same (bool): if pad by duplicating + pad_values (scalar or (ndarray, Tensor)): values to be padded if not pad_same + + Returns: + padded sequence (np.ndarray or torch.Tensor) + """ + assert isinstance(seq, (np.ndarray, torch.Tensor)) + assert pad_same or pad_values is not None + if pad_values is not None: + assert isinstance(pad_values, float) + repeat_func = np.repeat if isinstance(seq, np.ndarray) else torch.repeat_interleave + concat_func = np.concatenate if isinstance(seq, np.ndarray) else torch.cat + ones_like_func = np.ones_like if isinstance(seq, np.ndarray) else torch.ones_like + seq_dim = 1 if batched else 0 + + begin_pad = [] + end_pad = [] + + if padding[0] > 0: + pad = seq[[0]] if pad_same else ones_like_func(seq[[0]]) * pad_values + begin_pad.append(repeat_func(pad, padding[0], seq_dim)) + if padding[1] > 0: + pad = seq[[-1]] if pad_same else ones_like_func(seq[[-1]]) * pad_values + end_pad.append(repeat_func(pad, padding[1], seq_dim)) + + return concat_func(begin_pad + [seq] + end_pad, seq_dim) + + +def pad_sequence(seq, padding, batched=False, pad_same=True, pad_values=None): + """ + Pad a nested dictionary or list or tuple of sequence tensors in the time dimension (dimension 1). + + Args: + seq (dict or list or tuple): a possibly nested dictionary or list or tuple with tensors + of leading dimensions [B, T, ...] + padding (tuple): begin and end padding, e.g. [1, 1] pads both begin and end of the sequence by 1 + batched (bool): if sequence has the batch dimension + pad_same (bool): if pad by duplicating + pad_values (scalar or (ndarray, Tensor)): values to be padded if not pad_same + + Returns: + padded sequence (dict or list or tuple) + """ + return recursive_dict_list_tuple_apply( + seq, + { + torch.Tensor: + lambda x, p=padding, b=batched, ps=pad_same, pv=pad_values: pad_sequence_single(x, p, b, ps, pv), + np.ndarray: + lambda x, p=padding, b=batched, ps=pad_same, pv=pad_values: pad_sequence_single(x, p, b, ps, pv), + type(None): lambda x: x, + }, + ) + + +def assert_size_at_dim_single(x, size, dim, msg): + """ + Ensure that array or tensor @x has size @size in dim @dim. + + Args: + x (np.ndarray or torch.Tensor): input array or tensor + size (int): size that tensors should have at @dim + dim (int): dimension to check + msg (str): text to display if assertion fails + """ + assert x.shape[dim] == size, msg + + +def assert_size_at_dim(x, size, dim, msg): + """ + Ensure that arrays and tensors in nested dictionary or list or tuple have + size @size in dim @dim. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + size (int): size that tensors should have at @dim + dim (int): dimension to check + """ + map_tensor(x, lambda t, s=size, d=dim, m=msg: assert_size_at_dim_single(t, s, d, m)) + + +def get_shape(x): + """ + Get all shapes of arrays and tensors in nested dictionary or list or tuple. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + + Returns: + y (dict or list or tuple): new nested dict-list-tuple that contains each array or + tensor's shape + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: lambda x: x.shape, + np.ndarray: lambda x: x.shape, + type(None): lambda x: x, + }, + ) + + +def list_of_flat_dict_to_dict_of_list(list_of_dict): + """ + Helper function to go from a list of flat dictionaries to a dictionary of lists. + By "flat" we mean that none of the values are dictionaries, but are numpy arrays, + floats, etc. + + Args: + list_of_dict (list): list of flat dictionaries + + Returns: + dict_of_list (dict): dictionary of lists + """ + assert isinstance(list_of_dict, list) + dic = collections.OrderedDict() + for i in range(len(list_of_dict)): + for k in list_of_dict[i]: + if k not in dic: + dic[k] = [] + dic[k].append(list_of_dict[i][k]) + return dic + + +def flatten_nested_dict_list(d, parent_key="", sep="_", item_key=""): + """ + Flatten a nested dict or list to a list. + + For example, given a dict + { + a: 1 + b: { + c: 2 + } + c: 3 + } + + the function would return [(a, 1), (b_c, 2), (c, 3)] + + Args: + d (dict, list): a nested dict or list to be flattened + parent_key (str): recursion helper + sep (str): separator for nesting keys + item_key (str): recursion helper + Returns: + list: a list of (key, value) tuples + """ + items = [] + if isinstance(d, (tuple, list)): + new_key = parent_key + sep + item_key if len(parent_key) > 0 else item_key + for i, v in enumerate(d): + items.extend(flatten_nested_dict_list(v, new_key, sep=sep, item_key=str(i))) + return items + elif isinstance(d, dict): + new_key = parent_key + sep + item_key if len(parent_key) > 0 else item_key + for k, v in d.items(): + assert isinstance(k, str) + items.extend(flatten_nested_dict_list(v, new_key, sep=sep, item_key=k)) + return items + else: + new_key = parent_key + sep + item_key if len(parent_key) > 0 else item_key + return [(new_key, d)] + + +def time_distributed(inputs, op, activation=None, inputs_as_kwargs=False, inputs_as_args=False, **kwargs): + """ + Apply function @op to all tensors in nested dictionary or list or tuple @inputs in both the + batch (B) and time (T) dimension, where the tensors are expected to have shape [B, T, ...]. + Will do this by reshaping tensors to [B * T, ...], passing through the op, and then reshaping + outputs to [B, T, ...]. + + Args: + inputs (list or tuple or dict): a possibly nested dictionary or list or tuple with tensors + of leading dimensions [B, T, ...] + op: a layer op that accepts inputs + activation: activation to apply at the output + inputs_as_kwargs (bool): whether to feed input as a kwargs dict to the op + inputs_as_args (bool) whether to feed input as a args list to the op + kwargs (dict): other kwargs to supply to the op + + Returns: + outputs (dict or list or tuple): new nested dict-list-tuple with tensors of leading dimension [B, T]. + """ + batch_size, seq_len = flatten_nested_dict_list(inputs)[0][1].shape[:2] + inputs = join_dimensions(inputs, 0, 1) + if inputs_as_kwargs: + outputs = op(**inputs, **kwargs) + elif inputs_as_args: + outputs = op(*inputs, **kwargs) + else: + outputs = op(inputs, **kwargs) + + if activation is not None: + outputs = map_tensor(outputs, activation) + outputs = reshape_dimensions(outputs, begin_axis=0, end_axis=0, target_dims=(batch_size, seq_len)) + return outputs diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/diffusion/conditional_unet1d.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/diffusion/conditional_unet1d.py new file mode 100644 index 00000000..ea6616e1 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/diffusion/conditional_unet1d.py @@ -0,0 +1,278 @@ +from typing import Union +import logging +import torch +import torch.nn as nn +import einops +from einops.layers.torch import Rearrange + +from diffusion_policy.model.diffusion.conv1d_components import ( + Downsample1d, + Upsample1d, + Conv1dBlock, +) +from diffusion_policy.model.diffusion.positional_embedding import SinusoidalPosEmb + +logger = logging.getLogger(__name__) + + +class ConditionalResidualBlock1D(nn.Module): + + def __init__( + self, + in_channels, + out_channels, + cond_dim, + kernel_size=3, + n_groups=8, + cond_predict_scale=False, + ): + super().__init__() + + self.blocks = nn.ModuleList([ + Conv1dBlock(in_channels, out_channels, kernel_size, n_groups=n_groups), + Conv1dBlock(out_channels, out_channels, kernel_size, n_groups=n_groups), + ]) + + # FiLM modulation https://arxiv.org/abs/1709.07871 + # predicts per-channel scale and bias + cond_channels = out_channels + if cond_predict_scale: + cond_channels = out_channels * 2 + self.cond_predict_scale = cond_predict_scale + self.out_channels = out_channels + self.cond_encoder = nn.Sequential( + nn.Mish(), + nn.Linear(cond_dim, cond_channels), + Rearrange("batch t -> batch t 1"), + ) + + # make sure dimensions compatible + self.residual_conv = (nn.Conv1d(in_channels, out_channels, 1) if in_channels != out_channels else nn.Identity()) + + def forward(self, x, cond): + """ + x : [ batch_size x in_channels x horizon ] + cond : [ batch_size x cond_dim] + + returns: + out : [ batch_size x out_channels x horizon ] + """ + out = self.blocks[0](x) + embed = self.cond_encoder(cond) + if self.cond_predict_scale: + embed = embed.reshape(embed.shape[0], 2, self.out_channels, 1) + scale = embed[:, 0, ...] + bias = embed[:, 1, ...] + out = scale * out + bias + else: + out = out + embed + out = self.blocks[1](out) + out = out + self.residual_conv(x) + return out + + +class ConditionalUnet1D(nn.Module): + + def __init__( + self, + input_dim, + local_cond_dim=None, + global_cond_dim=None, + diffusion_step_embed_dim=256, + down_dims=[256, 512, 1024], + kernel_size=3, + n_groups=8, + cond_predict_scale=False, + ): + super().__init__() + all_dims = [input_dim] + list(down_dims) + start_dim = down_dims[0] + + dsed = diffusion_step_embed_dim + diffusion_step_encoder = nn.Sequential( + SinusoidalPosEmb(dsed), + nn.Linear(dsed, dsed * 4), + nn.Mish(), + nn.Linear(dsed * 4, dsed), + ) + cond_dim = dsed + if global_cond_dim is not None: + cond_dim += global_cond_dim + + in_out = list(zip(all_dims[:-1], all_dims[1:])) + + local_cond_encoder = None + if local_cond_dim is not None: + _, dim_out = in_out[0] + dim_in = local_cond_dim + local_cond_encoder = nn.ModuleList([ + # down encoder + ConditionalResidualBlock1D( + dim_in, + dim_out, + cond_dim=cond_dim, + kernel_size=kernel_size, + n_groups=n_groups, + cond_predict_scale=cond_predict_scale, + ), + # up encoder + ConditionalResidualBlock1D( + dim_in, + dim_out, + cond_dim=cond_dim, + kernel_size=kernel_size, + n_groups=n_groups, + cond_predict_scale=cond_predict_scale, + ), + ]) + + mid_dim = all_dims[-1] + self.mid_modules = nn.ModuleList([ + ConditionalResidualBlock1D( + mid_dim, + mid_dim, + cond_dim=cond_dim, + kernel_size=kernel_size, + n_groups=n_groups, + cond_predict_scale=cond_predict_scale, + ), + ConditionalResidualBlock1D( + mid_dim, + mid_dim, + cond_dim=cond_dim, + kernel_size=kernel_size, + n_groups=n_groups, + cond_predict_scale=cond_predict_scale, + ), + ]) + + down_modules = nn.ModuleList([]) + for ind, (dim_in, dim_out) in enumerate(in_out): + is_last = ind >= (len(in_out) - 1) + down_modules.append( + nn.ModuleList([ + ConditionalResidualBlock1D( + dim_in, + dim_out, + cond_dim=cond_dim, + kernel_size=kernel_size, + n_groups=n_groups, + cond_predict_scale=cond_predict_scale, + ), + ConditionalResidualBlock1D( + dim_out, + dim_out, + cond_dim=cond_dim, + kernel_size=kernel_size, + n_groups=n_groups, + cond_predict_scale=cond_predict_scale, + ), + Downsample1d(dim_out) if not is_last else nn.Identity(), + ])) + + up_modules = nn.ModuleList([]) + for ind, (dim_in, dim_out) in enumerate(reversed(in_out[1:])): + is_last = ind >= (len(in_out) - 1) + up_modules.append( + nn.ModuleList([ + ConditionalResidualBlock1D( + dim_out * 2, + dim_in, + cond_dim=cond_dim, + kernel_size=kernel_size, + n_groups=n_groups, + cond_predict_scale=cond_predict_scale, + ), + ConditionalResidualBlock1D( + dim_in, + dim_in, + cond_dim=cond_dim, + kernel_size=kernel_size, + n_groups=n_groups, + cond_predict_scale=cond_predict_scale, + ), + Upsample1d(dim_in) if not is_last else nn.Identity(), + ])) + + final_conv = nn.Sequential( + Conv1dBlock(start_dim, start_dim, kernel_size=kernel_size), + nn.Conv1d(start_dim, input_dim, 1), + ) + + self.diffusion_step_encoder = diffusion_step_encoder + self.local_cond_encoder = local_cond_encoder + self.up_modules = up_modules + self.down_modules = down_modules + self.final_conv = final_conv + + logger.info("number of parameters: %e", sum(p.numel() for p in self.parameters())) + + def forward(self, + sample: torch.Tensor, + timestep: Union[torch.Tensor, float, int], + local_cond=None, + global_cond=None, + **kwargs): + """ + x: (B,T,input_dim) + timestep: (B,) or int, diffusion step + local_cond: (B,T,local_cond_dim) + global_cond: (B,global_cond_dim) + output: (B,T,input_dim) + """ + sample = einops.rearrange(sample, "b h t -> b t h") + + # 1. time + timesteps = timestep + if not torch.is_tensor(timesteps): + # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can + timesteps = torch.tensor([timesteps], dtype=torch.long, device=sample.device) + elif torch.is_tensor(timesteps) and len(timesteps.shape) == 0: + timesteps = timesteps[None].to(sample.device) + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timesteps = timesteps.expand(sample.shape[0]) + + global_feature = self.diffusion_step_encoder(timesteps) + + if global_cond is not None: + global_feature = torch.cat([global_feature, global_cond], axis=-1) + + # encode local features + h_local = list() + if local_cond is not None: + local_cond = einops.rearrange(local_cond, "b h t -> b t h") + resnet, resnet2 = self.local_cond_encoder + x = resnet(local_cond, global_feature) + h_local.append(x) + x = resnet2(local_cond, global_feature) + h_local.append(x) + + x = sample + h = [] + for idx, (resnet, resnet2, downsample) in enumerate(self.down_modules): + x = resnet(x, global_feature) + if idx == 0 and len(h_local) > 0: + x = x + h_local[0] + x = resnet2(x, global_feature) + h.append(x) + x = downsample(x) + + for mid_module in self.mid_modules: + x = mid_module(x, global_feature) + + for idx, (resnet, resnet2, upsample) in enumerate(self.up_modules): + x = torch.cat((x, h.pop()), dim=1) + x = resnet(x, global_feature) + # The correct condition should be: + # if idx == (len(self.up_modules)-1) and len(h_local) > 0: + # However this change will break compatibility with published checkpoints. + # Therefore it is left as a comment. + if idx == len(self.up_modules) and len(h_local) > 0: + x = x + h_local[1] + x = resnet2(x, global_feature) + x = upsample(x) + + x = self.final_conv(x) + + x = einops.rearrange(x, "b t h -> b h t") + return x diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/diffusion/conv1d_components.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/diffusion/conv1d_components.py new file mode 100644 index 00000000..163ed05e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/diffusion/conv1d_components.py @@ -0,0 +1,51 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +# from einops.layers.torch import Rearrange + + +class Downsample1d(nn.Module): + + def __init__(self, dim): + super().__init__() + self.conv = nn.Conv1d(dim, dim, 3, 2, 1) + + def forward(self, x): + return self.conv(x) + + +class Upsample1d(nn.Module): + + def __init__(self, dim): + super().__init__() + self.conv = nn.ConvTranspose1d(dim, dim, 4, 2, 1) + + def forward(self, x): + return self.conv(x) + + +class Conv1dBlock(nn.Module): + """ + Conv1d --> GroupNorm --> Mish + """ + + def __init__(self, inp_channels, out_channels, kernel_size, n_groups=8): + super().__init__() + + self.block = nn.Sequential( + nn.Conv1d(inp_channels, out_channels, kernel_size, padding=kernel_size // 2), + # Rearrange('batch channels horizon -> batch channels 1 horizon'), + nn.GroupNorm(n_groups, out_channels), + # Rearrange('batch channels 1 horizon -> batch channels horizon'), + nn.Mish(), + ) + + def forward(self, x): + return self.block(x) + + +def test(): + cb = Conv1dBlock(256, 128, kernel_size=3) + x = torch.zeros((1, 256, 16)) + o = cb(x) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/diffusion/ema_model.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/diffusion/ema_model.py new file mode 100644 index 00000000..c6835f75 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/diffusion/ema_model.py @@ -0,0 +1,89 @@ +import copy +import torch +from torch.nn.modules.batchnorm import _BatchNorm + + +class EMAModel: + """ + Exponential Moving Average of models weights + """ + + def __init__( + self, + model, + update_after_step=0, + inv_gamma=1.0, + power=2 / 3, + min_value=0.0, + max_value=0.9999, + ): + """ + @crowsonkb's notes on EMA Warmup: + If gamma=1 and power=1, implements a simple average. gamma=1, power=2/3 are good values for models you plan + to train for a million or more steps (reaches decay factor 0.999 at 31.6K steps, 0.9999 at 1M steps), + gamma=1, power=3/4 for models you plan to train for less (reaches decay factor 0.999 at 10K steps, 0.9999 + at 215.4k steps). + Args: + inv_gamma (float): Inverse multiplicative factor of EMA warmup. Default: 1. + power (float): Exponential factor of EMA warmup. Default: 2/3. + min_value (float): The minimum EMA decay rate. Default: 0. + """ + + self.averaged_model = model + self.averaged_model.eval() + self.averaged_model.requires_grad_(False) + + self.update_after_step = update_after_step + self.inv_gamma = inv_gamma + self.power = power + self.min_value = min_value + self.max_value = max_value + + self.decay = 0.0 + self.optimization_step = 0 + + def get_decay(self, optimization_step): + """ + Compute the decay factor for the exponential moving average. + """ + step = max(0, optimization_step - self.update_after_step - 1) + value = 1 - (1 + step / self.inv_gamma)**-self.power + + if step <= 0: + return 0.0 + + return max(self.min_value, min(value, self.max_value)) + + @torch.no_grad() + def step(self, new_model): + self.decay = self.get_decay(self.optimization_step) + + # old_all_dataptrs = set() + # for param in new_model.parameters(): + # data_ptr = param.data_ptr() + # if data_ptr != 0: + # old_all_dataptrs.add(data_ptr) + + all_dataptrs = set() + for module, ema_module in zip(new_model.modules(), self.averaged_model.modules()): + for param, ema_param in zip(module.parameters(recurse=False), ema_module.parameters(recurse=False)): + # iterative over immediate parameters only. + if isinstance(param, dict): + raise RuntimeError("Dict parameter not supported") + + # data_ptr = param.data_ptr() + # if data_ptr != 0: + # all_dataptrs.add(data_ptr) + + if isinstance(module, _BatchNorm): + # skip batchnorms + ema_param.copy_(param.to(dtype=ema_param.dtype).data) + elif not param.requires_grad: + ema_param.copy_(param.to(dtype=ema_param.dtype).data) + else: + ema_param.mul_(self.decay) + ema_param.add_(param.data.to(dtype=ema_param.dtype), alpha=1 - self.decay) + + # verify that iterating over module and then parameters is identical to parameters recursively. + # assert old_all_dataptrs == all_dataptrs + self.optimization_step += 1 diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/diffusion/mask_generator.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/diffusion/mask_generator.py new file mode 100644 index 00000000..8c149314 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/diffusion/mask_generator.py @@ -0,0 +1,225 @@ +from typing import Sequence, Optional +import torch +from torch import nn +from diffusion_policy.model.common.module_attr_mixin import ModuleAttrMixin + + +def get_intersection_slice_mask(shape: tuple, dim_slices: Sequence[slice], device: Optional[torch.device] = None): + assert len(shape) == len(dim_slices) + mask = torch.zeros(size=shape, dtype=torch.bool, device=device) + mask[dim_slices] = True + return mask + + +def get_union_slice_mask(shape: tuple, dim_slices: Sequence[slice], device: Optional[torch.device] = None): + assert len(shape) == len(dim_slices) + mask = torch.zeros(size=shape, dtype=torch.bool, device=device) + for i in range(len(dim_slices)): + this_slices = [slice(None)] * len(shape) + this_slices[i] = dim_slices[i] + mask[this_slices] = True + return mask + + +class DummyMaskGenerator(ModuleAttrMixin): + + def __init__(self): + super().__init__() + + @torch.no_grad() + def forward(self, shape): + device = self.device + mask = torch.ones(size=shape, dtype=torch.bool, device=device) + return mask + + +class LowdimMaskGenerator(ModuleAttrMixin): + + def __init__( + self, + action_dim, + obs_dim, + # obs mask setup + max_n_obs_steps=2, + fix_obs_steps=True, + # action mask + action_visible=False, + ): + super().__init__() + self.action_dim = action_dim + self.obs_dim = obs_dim + self.max_n_obs_steps = max_n_obs_steps + self.fix_obs_steps = fix_obs_steps + self.action_visible = action_visible + + @torch.no_grad() + def forward(self, shape, seed=None): + device = self.device + B, T, D = shape + assert D == (self.action_dim + self.obs_dim) + + # create all tensors on this device + rng = torch.Generator(device=device) + if seed is not None: + rng = rng.manual_seed(seed) + + # generate dim mask + dim_mask = torch.zeros(size=shape, dtype=torch.bool, device=device) + is_action_dim = dim_mask.clone() + is_action_dim[..., :self.action_dim] = True + is_obs_dim = ~is_action_dim + + # generate obs mask + if self.fix_obs_steps: + obs_steps = torch.full((B, ), fill_value=self.max_n_obs_steps, device=device) + else: + obs_steps = torch.randint( + low=1, + high=self.max_n_obs_steps + 1, + size=(B, ), + generator=rng, + device=device, + ) + + steps = torch.arange(0, T, device=device).reshape(1, T).expand(B, T) + obs_mask = (steps.T < obs_steps).T.reshape(B, T, 1).expand(B, T, D) + obs_mask = obs_mask & is_obs_dim + + # generate action mask + if self.action_visible: + action_steps = torch.maximum( + obs_steps - 1, + torch.tensor(0, dtype=obs_steps.dtype, device=obs_steps.device), + ) + action_mask = (steps.T < action_steps).T.reshape(B, T, 1).expand(B, T, D) + action_mask = action_mask & is_action_dim + + mask = obs_mask + if self.action_visible: + mask = mask | action_mask + + return mask + + +class KeypointMaskGenerator(ModuleAttrMixin): + + def __init__( + self, + # dimensions + action_dim, + keypoint_dim, + # obs mask setup + max_n_obs_steps=2, + fix_obs_steps=True, + # keypoint mask setup + keypoint_visible_rate=0.7, + time_independent=False, + # action mask + action_visible=False, + context_dim=0, # dim for context + n_context_steps=1, + ): + super().__init__() + self.action_dim = action_dim + self.keypoint_dim = keypoint_dim + self.context_dim = context_dim + self.max_n_obs_steps = max_n_obs_steps + self.fix_obs_steps = fix_obs_steps + self.keypoint_visible_rate = keypoint_visible_rate + self.time_independent = time_independent + self.action_visible = action_visible + self.n_context_steps = n_context_steps + + @torch.no_grad() + def forward(self, shape, seed=None): + device = self.device + B, T, D = shape + all_keypoint_dims = D - self.action_dim - self.context_dim + n_keypoints = all_keypoint_dims // self.keypoint_dim + + # create all tensors on this device + rng = torch.Generator(device=device) + if seed is not None: + rng = rng.manual_seed(seed) + + # generate dim mask + dim_mask = torch.zeros(size=shape, dtype=torch.bool, device=device) + is_action_dim = dim_mask.clone() + is_action_dim[..., :self.action_dim] = True + is_context_dim = dim_mask.clone() + if self.context_dim > 0: + is_context_dim[..., -self.context_dim:] = True + is_obs_dim = ~(is_action_dim | is_context_dim) + # assumption trajectory=cat([action, keypoints, context], dim=-1) + + # generate obs mask + if self.fix_obs_steps: + obs_steps = torch.full((B, ), fill_value=self.max_n_obs_steps, device=device) + else: + obs_steps = torch.randint( + low=1, + high=self.max_n_obs_steps + 1, + size=(B, ), + generator=rng, + device=device, + ) + + steps = torch.arange(0, T, device=device).reshape(1, T).expand(B, T) + obs_mask = (steps.T < obs_steps).T.reshape(B, T, 1).expand(B, T, D) + obs_mask = obs_mask & is_obs_dim + + # generate action mask + if self.action_visible: + action_steps = torch.maximum( + obs_steps - 1, + torch.tensor(0, dtype=obs_steps.dtype, device=obs_steps.device), + ) + action_mask = (steps.T < action_steps).T.reshape(B, T, 1).expand(B, T, D) + action_mask = action_mask & is_action_dim + + # generate keypoint mask + if self.time_independent: + visible_kps = (torch.rand(size=(B, T, n_keypoints), generator=rng, device=device) + < self.keypoint_visible_rate) + visible_dims = torch.repeat_interleave(visible_kps, repeats=self.keypoint_dim, dim=-1) + visible_dims_mask = torch.cat( + [ + torch.ones((B, T, self.action_dim), dtype=torch.bool, device=device), + visible_dims, + torch.ones((B, T, self.context_dim), dtype=torch.bool, device=device), + ], + axis=-1, + ) + keypoint_mask = visible_dims_mask + else: + visible_kps = (torch.rand(size=(B, n_keypoints), generator=rng, device=device) < self.keypoint_visible_rate) + visible_dims = torch.repeat_interleave(visible_kps, repeats=self.keypoint_dim, dim=-1) + visible_dims_mask = torch.cat( + [ + torch.ones((B, self.action_dim), dtype=torch.bool, device=device), + visible_dims, + torch.ones((B, self.context_dim), dtype=torch.bool, device=device), + ], + axis=-1, + ) + keypoint_mask = visible_dims_mask.reshape(B, 1, D).expand(B, T, D) + keypoint_mask = keypoint_mask & is_obs_dim + + # generate context mask + context_mask = is_context_dim.clone() + context_mask[:, self.n_context_steps:, :] = False + + mask = obs_mask & keypoint_mask + if self.action_visible: + mask = mask | action_mask + if self.context_dim > 0: + mask = mask | context_mask + + return mask + + +def test(): + # kmg = KeypointMaskGenerator(2,2, random_obs_steps=True) + # self = KeypointMaskGenerator(2,2,context_dim=2, action_visible=True) + # self = KeypointMaskGenerator(2,2,context_dim=0, action_visible=True) + self = LowdimMaskGenerator(2, 20, max_n_obs_steps=3, action_visible=True) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/diffusion/positional_embedding.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/diffusion/positional_embedding.py new file mode 100644 index 00000000..1b1d646d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/diffusion/positional_embedding.py @@ -0,0 +1,19 @@ +import math +import torch +import torch.nn as nn + + +class SinusoidalPosEmb(nn.Module): + + def __init__(self, dim): + super().__init__() + self.dim = dim + + def forward(self, x): + device = x.device + half_dim = self.dim // 2 + emb = math.log(10000) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, device=device) * -emb) + emb = x[:, None] * emb[None, :] + emb = torch.cat((emb.sin(), emb.cos()), dim=-1) + return emb diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/diffusion/transformer_for_diffusion.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/diffusion/transformer_for_diffusion.py new file mode 100644 index 00000000..7b410713 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/diffusion/transformer_for_diffusion.py @@ -0,0 +1,391 @@ +from typing import Union, Optional, Tuple +import logging +import torch +import torch.nn as nn +from diffusion_policy.model.diffusion.positional_embedding import SinusoidalPosEmb +from diffusion_policy.model.common.module_attr_mixin import ModuleAttrMixin + +logger = logging.getLogger(__name__) + + +class TransformerForDiffusion(ModuleAttrMixin): + + def __init__( + self, + input_dim: int, + output_dim: int, + horizon: int, + n_obs_steps: int = None, + cond_dim: int = 0, + n_layer: int = 12, + n_head: int = 12, + n_emb: int = 768, + p_drop_emb: float = 0.1, + p_drop_attn: float = 0.1, + causal_attn: bool = False, + time_as_cond: bool = True, + obs_as_cond: bool = False, + n_cond_layers: int = 0, + ) -> None: + super().__init__() + + # compute number of tokens for main trunk and condition encoder + if n_obs_steps is None: + n_obs_steps = horizon + + T = horizon + T_cond = 1 + if not time_as_cond: + T += 1 + T_cond -= 1 + obs_as_cond = cond_dim > 0 + if obs_as_cond: + assert time_as_cond + T_cond += n_obs_steps + + # input embedding stem + self.input_emb = nn.Linear(input_dim, n_emb) + self.pos_emb = nn.Parameter(torch.zeros(1, T, n_emb)) + self.drop = nn.Dropout(p_drop_emb) + + # cond encoder + self.time_emb = SinusoidalPosEmb(n_emb) + self.cond_obs_emb = None + + if obs_as_cond: + self.cond_obs_emb = nn.Linear(cond_dim, n_emb) + + self.cond_pos_emb = None + self.encoder = None + self.decoder = None + encoder_only = False + if T_cond > 0: + self.cond_pos_emb = nn.Parameter(torch.zeros(1, T_cond, n_emb)) + if n_cond_layers > 0: + encoder_layer = nn.TransformerEncoderLayer( + d_model=n_emb, + nhead=n_head, + dim_feedforward=4 * n_emb, + dropout=p_drop_attn, + activation="gelu", + batch_first=True, + norm_first=True, + ) + self.encoder = nn.TransformerEncoder(encoder_layer=encoder_layer, num_layers=n_cond_layers) + else: + self.encoder = nn.Sequential(nn.Linear(n_emb, 4 * n_emb), nn.Mish(), nn.Linear(4 * n_emb, n_emb)) + # decoder + decoder_layer = nn.TransformerDecoderLayer( + d_model=n_emb, + nhead=n_head, + dim_feedforward=4 * n_emb, + dropout=p_drop_attn, + activation="gelu", + batch_first=True, + norm_first=True, # important for stability + ) + self.decoder = nn.TransformerDecoder(decoder_layer=decoder_layer, num_layers=n_layer) + else: + # encoder only BERT + encoder_only = True + + encoder_layer = nn.TransformerEncoderLayer( + d_model=n_emb, + nhead=n_head, + dim_feedforward=4 * n_emb, + dropout=p_drop_attn, + activation="gelu", + batch_first=True, + norm_first=True, + ) + self.encoder = nn.TransformerEncoder(encoder_layer=encoder_layer, num_layers=n_layer) + + # attention mask + if causal_attn: + # causal mask to ensure that attention is only applied to the left in the input sequence + # torch.nn.Transformer uses additive mask as opposed to multiplicative mask in minGPT + # therefore, the upper triangle should be -inf and others (including diag) should be 0. + sz = T + mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1) + mask = (mask.float().masked_fill(mask == 0, float("-inf")).masked_fill(mask == 1, float(0.0))) + self.register_buffer("mask", mask) + + if time_as_cond and obs_as_cond: + S = T_cond + t, s = torch.meshgrid(torch.arange(T), torch.arange(S), indexing="ij") + mask = t >= (s - 1) # add one dimension since time is the first token in cond + mask = (mask.float().masked_fill(mask == 0, float("-inf")).masked_fill(mask == 1, float(0.0))) + self.register_buffer("memory_mask", mask) + else: + self.memory_mask = None + else: + self.mask = None + self.memory_mask = None + + # decoder head + self.ln_f = nn.LayerNorm(n_emb) + self.head = nn.Linear(n_emb, output_dim) + + # constants + self.T = T + self.T_cond = T_cond + self.horizon = horizon + self.time_as_cond = time_as_cond + self.obs_as_cond = obs_as_cond + self.encoder_only = encoder_only + + # init + self.apply(self._init_weights) + logger.info("number of parameters: %e", sum(p.numel() for p in self.parameters())) + + def _init_weights(self, module): + ignore_types = ( + nn.Dropout, + SinusoidalPosEmb, + nn.TransformerEncoderLayer, + nn.TransformerDecoderLayer, + nn.TransformerEncoder, + nn.TransformerDecoder, + nn.ModuleList, + nn.Mish, + nn.Sequential, + ) + if isinstance(module, (nn.Linear, nn.Embedding)): + torch.nn.init.normal_(module.weight, mean=0.0, std=0.02) + if isinstance(module, nn.Linear) and module.bias is not None: + torch.nn.init.zeros_(module.bias) + elif isinstance(module, nn.MultiheadAttention): + weight_names = [ + "in_proj_weight", + "q_proj_weight", + "k_proj_weight", + "v_proj_weight", + ] + for name in weight_names: + weight = getattr(module, name) + if weight is not None: + torch.nn.init.normal_(weight, mean=0.0, std=0.02) + + bias_names = ["in_proj_bias", "bias_k", "bias_v"] + for name in bias_names: + bias = getattr(module, name) + if bias is not None: + torch.nn.init.zeros_(bias) + elif isinstance(module, nn.LayerNorm): + torch.nn.init.zeros_(module.bias) + torch.nn.init.ones_(module.weight) + elif isinstance(module, TransformerForDiffusion): + torch.nn.init.normal_(module.pos_emb, mean=0.0, std=0.02) + if module.cond_obs_emb is not None: + torch.nn.init.normal_(module.cond_pos_emb, mean=0.0, std=0.02) + elif isinstance(module, ignore_types): + # no param + pass + else: + raise RuntimeError("Unaccounted module {}".format(module)) + + def get_optim_groups(self, weight_decay: float = 1e-3): + """ + This long function is unfortunately doing something very simple and is being very defensive: + We are separating out all parameters of the model into two buckets: those that will experience + weight decay for regularization and those that won't (biases, and layernorm/embedding weights). + We are then returning the PyTorch optimizer object. + """ + + # separate out all parameters to those that will and won't experience regularizing weight decay + decay = set() + no_decay = set() + whitelist_weight_modules = (torch.nn.Linear, torch.nn.MultiheadAttention) + blacklist_weight_modules = (torch.nn.LayerNorm, torch.nn.Embedding) + for mn, m in self.named_modules(): + for pn, p in m.named_parameters(): + fpn = "%s.%s" % (mn, pn) if mn else pn # full param name + + if pn.endswith("bias"): + # all biases will not be decayed + no_decay.add(fpn) + elif pn.startswith("bias"): + # MultiheadAttention bias starts with "bias" + no_decay.add(fpn) + elif pn.endswith("weight") and isinstance(m, whitelist_weight_modules): + # weights of whitelist modules will be weight decayed + decay.add(fpn) + elif pn.endswith("weight") and isinstance(m, blacklist_weight_modules): + # weights of blacklist modules will NOT be weight decayed + no_decay.add(fpn) + + # special case the position embedding parameter in the root GPT module as not decayed + no_decay.add("pos_emb") + no_decay.add("_dummy_variable") + if self.cond_pos_emb is not None: + no_decay.add("cond_pos_emb") + + # validate that we considered every parameter + param_dict = {pn: p for pn, p in self.named_parameters()} + inter_params = decay & no_decay + union_params = decay | no_decay + assert (len(inter_params) == 0), "parameters %s made it into both decay/no_decay sets!" % (str(inter_params), ) + assert (len(param_dict.keys() - + union_params) == 0), "parameters %s were not separated into either decay/no_decay set!" % ( + str(param_dict.keys() - union_params), ) + + # create the pytorch optimizer object + optim_groups = [ + { + "params": [param_dict[pn] for pn in sorted(list(decay))], + "weight_decay": weight_decay, + }, + { + "params": [param_dict[pn] for pn in sorted(list(no_decay))], + "weight_decay": 0.0, + }, + ] + return optim_groups + + def configure_optimizers( + self, + learning_rate: float = 1e-4, + weight_decay: float = 1e-3, + betas: Tuple[float, float] = (0.9, 0.95), + ): + optim_groups = self.get_optim_groups(weight_decay=weight_decay) + optimizer = torch.optim.AdamW(optim_groups, lr=learning_rate, betas=betas) + return optimizer + + def forward(self, + sample: torch.Tensor, + timestep: Union[torch.Tensor, float, int], + cond: Optional[torch.Tensor] = None, + **kwargs): + """ + x: (B,T,input_dim) + timestep: (B,) or int, diffusion step + cond: (B,T',cond_dim) + output: (B,T,input_dim) + """ + # 1. time + timesteps = timestep + if not torch.is_tensor(timesteps): + # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can + timesteps = torch.tensor([timesteps], dtype=torch.long, device=sample.device) + elif torch.is_tensor(timesteps) and len(timesteps.shape) == 0: + timesteps = timesteps[None].to(sample.device) + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timesteps = timesteps.expand(sample.shape[0]) + time_emb = self.time_emb(timesteps).unsqueeze(1) + # (B,1,n_emb) + + # process input + input_emb = self.input_emb(sample) + + if self.encoder_only: + # BERT + token_embeddings = torch.cat([time_emb, input_emb], dim=1) + t = token_embeddings.shape[1] + position_embeddings = self.pos_emb[:, :t, :] # each position maps to a (learnable) vector + x = self.drop(token_embeddings + position_embeddings) + # (B,T+1,n_emb) + x = self.encoder(src=x, mask=self.mask) + # (B,T+1,n_emb) + x = x[:, 1:, :] + # (B,T,n_emb) + else: + # encoder + cond_embeddings = time_emb + if self.obs_as_cond: + cond_obs_emb = self.cond_obs_emb(cond) + # (B,To,n_emb) + cond_embeddings = torch.cat([cond_embeddings, cond_obs_emb], dim=1) + tc = cond_embeddings.shape[1] + position_embeddings = self.cond_pos_emb[:, :tc, :] # each position maps to a (learnable) vector + x = self.drop(cond_embeddings + position_embeddings) + x = self.encoder(x) + memory = x + # (B,T_cond,n_emb) + + # decoder + token_embeddings = input_emb + t = token_embeddings.shape[1] + position_embeddings = self.pos_emb[:, :t, :] # each position maps to a (learnable) vector + x = self.drop(token_embeddings + position_embeddings) + # (B,T,n_emb) + x = self.decoder(tgt=x, memory=memory, tgt_mask=self.mask, memory_mask=self.memory_mask) + # (B,T,n_emb) + + # head + x = self.ln_f(x) + x = self.head(x) + # (B,T,n_out) + return x + + +def test(): + # GPT with time embedding + transformer = TransformerForDiffusion( + input_dim=16, + output_dim=16, + horizon=8, + n_obs_steps=4, + # cond_dim=10, + causal_attn=True, + # time_as_cond=False, + # n_cond_layers=4 + ) + opt = transformer.configure_optimizers() + + timestep = torch.tensor(0) + sample = torch.zeros((4, 8, 16)) + out = transformer(sample, timestep) + + # GPT with time embedding and obs cond + transformer = TransformerForDiffusion( + input_dim=16, + output_dim=16, + horizon=8, + n_obs_steps=4, + cond_dim=10, + causal_attn=True, + # time_as_cond=False, + # n_cond_layers=4 + ) + opt = transformer.configure_optimizers() + + timestep = torch.tensor(0) + sample = torch.zeros((4, 8, 16)) + cond = torch.zeros((4, 4, 10)) + out = transformer(sample, timestep, cond) + + # GPT with time embedding and obs cond and encoder + transformer = TransformerForDiffusion( + input_dim=16, + output_dim=16, + horizon=8, + n_obs_steps=4, + cond_dim=10, + causal_attn=True, + # time_as_cond=False, + n_cond_layers=4, + ) + opt = transformer.configure_optimizers() + + timestep = torch.tensor(0) + sample = torch.zeros((4, 8, 16)) + cond = torch.zeros((4, 4, 10)) + out = transformer(sample, timestep, cond) + + # BERT with time embedding token + transformer = TransformerForDiffusion( + input_dim=16, + output_dim=16, + horizon=8, + n_obs_steps=4, + # cond_dim=10, + # causal_attn=True, + time_as_cond=False, + # n_cond_layers=4 + ) + opt = transformer.configure_optimizers() + + timestep = torch.tensor(0) + sample = torch.zeros((4, 8, 16)) + out = transformer(sample, timestep) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/vision/crop_randomizer.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/vision/crop_randomizer.py new file mode 100644 index 00000000..7124fce3 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/vision/crop_randomizer.py @@ -0,0 +1,298 @@ +import torch +import torch.nn as nn +import torchvision.transforms.functional as ttf +import diffusion_policy.model.common.tensor_util as tu + + +class CropRandomizer(nn.Module): + """ + Randomly sample crops at input, and then average across crop features at output. + """ + + def __init__( + self, + input_shape, + crop_height, + crop_width, + num_crops=1, + pos_enc=False, + ): + """ + Args: + input_shape (tuple, list): shape of input (not including batch dimension) + crop_height (int): crop height + crop_width (int): crop width + num_crops (int): number of random crops to take + pos_enc (bool): if True, add 2 channels to the output to encode the spatial + location of the cropped pixels in the source image + """ + super().__init__() + + assert len(input_shape) == 3 # (C, H, W) + assert crop_height < input_shape[1] + assert crop_width < input_shape[2] + + self.input_shape = input_shape + self.crop_height = crop_height + self.crop_width = crop_width + self.num_crops = num_crops + self.pos_enc = pos_enc + + def output_shape_in(self, input_shape=None): + """ + Function to compute output shape from inputs to this module. Corresponds to + the @forward_in operation, where raw inputs (usually observation modalities) + are passed in. + + Args: + input_shape (iterable of int): shape of input. Does not include batch dimension. + Some modules may not need this argument, if their output does not depend + on the size of the input, or if they assume fixed size input. + + Returns: + out_shape ([int]): list of integers corresponding to output shape + """ + + # outputs are shape (C, CH, CW), or maybe C + 2 if using position encoding, because + # the number of crops are reshaped into the batch dimension, increasing the batch + # size from B to B * N + out_c = self.input_shape[0] + 2 if self.pos_enc else self.input_shape[0] + return [out_c, self.crop_height, self.crop_width] + + def output_shape_out(self, input_shape=None): + """ + Function to compute output shape from inputs to this module. Corresponds to + the @forward_out operation, where processed inputs (usually encoded observation + modalities) are passed in. + + Args: + input_shape (iterable of int): shape of input. Does not include batch dimension. + Some modules may not need this argument, if their output does not depend + on the size of the input, or if they assume fixed size input. + + Returns: + out_shape ([int]): list of integers corresponding to output shape + """ + + # since the forward_out operation splits [B * N, ...] -> [B, N, ...] + # and then pools to result in [B, ...], only the batch dimension changes, + # and so the other dimensions retain their shape. + return list(input_shape) + + def forward_in(self, inputs): + """ + Samples N random crops for each input in the batch, and then reshapes + inputs to [B * N, ...]. + """ + assert len(inputs.shape) >= 3 # must have at least (C, H, W) dimensions + if self.training: + # generate random crops + out, _ = sample_random_image_crops( + images=inputs, + crop_height=self.crop_height, + crop_width=self.crop_width, + num_crops=self.num_crops, + pos_enc=self.pos_enc, + ) + # [B, N, ...] -> [B * N, ...] + return tu.join_dimensions(out, 0, 1) + else: + # take center crop during eval + out = ttf.center_crop(img=inputs, output_size=(self.crop_height, self.crop_width)) + if self.num_crops > 1: + B, C, H, W = out.shape + out = (out.unsqueeze(1).expand(B, self.num_crops, C, H, W).reshape(-1, C, H, W)) + # [B * N, ...] + return out + + def forward_out(self, inputs): + """ + Splits the outputs from shape [B * N, ...] -> [B, N, ...] and then average across N + to result in shape [B, ...] to make sure the network output is consistent with + what would have happened if there were no randomization. + """ + if self.num_crops <= 1: + return inputs + else: + batch_size = inputs.shape[0] // self.num_crops + out = tu.reshape_dimensions( + inputs, + begin_axis=0, + end_axis=0, + target_dims=(batch_size, self.num_crops), + ) + return out.mean(dim=1) + + def forward(self, inputs): + return self.forward_in(inputs) + + def __repr__(self): + """Pretty print network.""" + header = "{}".format(str(self.__class__.__name__)) + msg = header + "(input_shape={}, crop_size=[{}, {}], num_crops={})".format(self.input_shape, self.crop_height, + self.crop_width, self.num_crops) + return msg + + +def crop_image_from_indices(images, crop_indices, crop_height, crop_width): + """ + Crops images at the locations specified by @crop_indices. Crops will be + taken across all channels. + + Args: + images (torch.Tensor): batch of images of shape [..., C, H, W] + + crop_indices (torch.Tensor): batch of indices of shape [..., N, 2] where + N is the number of crops to take per image and each entry corresponds + to the pixel height and width of where to take the crop. Note that + the indices can also be of shape [..., 2] if only 1 crop should + be taken per image. Leading dimensions must be consistent with + @images argument. Each index specifies the top left of the crop. + Values must be in range [0, H - CH - 1] x [0, W - CW - 1] where + H and W are the height and width of @images and CH and CW are + @crop_height and @crop_width. + + crop_height (int): height of crop to take + + crop_width (int): width of crop to take + + Returns: + crops (torch.Tesnor): cropped images of shape [..., C, @crop_height, @crop_width] + """ + + # make sure length of input shapes is consistent + assert crop_indices.shape[-1] == 2 + ndim_im_shape = len(images.shape) + ndim_indices_shape = len(crop_indices.shape) + assert (ndim_im_shape == ndim_indices_shape + 1) or (ndim_im_shape == ndim_indices_shape + 2) + + # maybe pad so that @crop_indices is shape [..., N, 2] + is_padded = False + if ndim_im_shape == ndim_indices_shape + 2: + crop_indices = crop_indices.unsqueeze(-2) + is_padded = True + + # make sure leading dimensions between images and indices are consistent + assert images.shape[:-3] == crop_indices.shape[:-2] + + device = images.device + image_c, image_h, image_w = images.shape[-3:] + num_crops = crop_indices.shape[-2] + + # make sure @crop_indices are in valid range + assert (crop_indices[..., 0] >= 0).all().item() + assert (crop_indices[..., 0] < (image_h - crop_height)).all().item() + assert (crop_indices[..., 1] >= 0).all().item() + assert (crop_indices[..., 1] < (image_w - crop_width)).all().item() + + # convert each crop index (ch, cw) into a list of pixel indices that correspond to the entire window. + + # 2D index array with columns [0, 1, ..., CH - 1] and shape [CH, CW] + crop_ind_grid_h = torch.arange(crop_height).to(device) + crop_ind_grid_h = tu.unsqueeze_expand_at(crop_ind_grid_h, size=crop_width, dim=-1) + # 2D index array with rows [0, 1, ..., CW - 1] and shape [CH, CW] + crop_ind_grid_w = torch.arange(crop_width).to(device) + crop_ind_grid_w = tu.unsqueeze_expand_at(crop_ind_grid_w, size=crop_height, dim=0) + # combine into shape [CH, CW, 2] + crop_in_grid = torch.cat((crop_ind_grid_h.unsqueeze(-1), crop_ind_grid_w.unsqueeze(-1)), dim=-1) + + # Add above grid with the offset index of each sampled crop to get 2d indices for each crop. + # After broadcasting, this will be shape [..., N, CH, CW, 2] and each crop has a [CH, CW, 2] + # shape array that tells us which pixels from the corresponding source image to grab. + grid_reshape = [1] * len(crop_indices.shape[:-1]) + [crop_height, crop_width, 2] + all_crop_inds = crop_indices.unsqueeze(-2).unsqueeze(-2) + crop_in_grid.reshape(grid_reshape) + + # For using @torch.gather, convert to flat indices from 2D indices, and also + # repeat across the channel dimension. To get flat index of each pixel to grab for + # each sampled crop, we just use the mapping: ind = h_ind * @image_w + w_ind + all_crop_inds = (all_crop_inds[..., 0] * image_w + all_crop_inds[..., 1]) # shape [..., N, CH, CW] + all_crop_inds = tu.unsqueeze_expand_at(all_crop_inds, size=image_c, dim=-3) # shape [..., N, C, CH, CW] + all_crop_inds = tu.flatten(all_crop_inds, begin_axis=-2) # shape [..., N, C, CH * CW] + + # Repeat and flatten the source images -> [..., N, C, H * W] and then use gather to index with crop pixel inds + images_to_crop = tu.unsqueeze_expand_at(images, size=num_crops, dim=-4) + images_to_crop = tu.flatten(images_to_crop, begin_axis=-2) + crops = torch.gather(images_to_crop, dim=-1, index=all_crop_inds) + # [..., N, C, CH * CW] -> [..., N, C, CH, CW] + reshape_axis = len(crops.shape) - 1 + crops = tu.reshape_dimensions( + crops, + begin_axis=reshape_axis, + end_axis=reshape_axis, + target_dims=(crop_height, crop_width), + ) + + if is_padded: + # undo padding -> [..., C, CH, CW] + crops = crops.squeeze(-4) + return crops + + +def sample_random_image_crops(images, crop_height, crop_width, num_crops, pos_enc=False): + """ + For each image, randomly sample @num_crops crops of size (@crop_height, @crop_width), from + @images. + + Args: + images (torch.Tensor): batch of images of shape [..., C, H, W] + + crop_height (int): height of crop to take + + crop_width (int): width of crop to take + + num_crops (n): number of crops to sample + + pos_enc (bool): if True, also add 2 channels to the outputs that gives a spatial + encoding of the original source pixel locations. This means that the + output crops will contain information about where in the source image + it was sampled from. + + Returns: + crops (torch.Tensor): crops of shape (..., @num_crops, C, @crop_height, @crop_width) + if @pos_enc is False, otherwise (..., @num_crops, C + 2, @crop_height, @crop_width) + + crop_inds (torch.Tensor): sampled crop indices of shape (..., N, 2) + """ + device = images.device + + # maybe add 2 channels of spatial encoding to the source image + source_im = images + if pos_enc: + # spatial encoding [y, x] in [0, 1] + h, w = source_im.shape[-2:] + pos_y, pos_x = torch.meshgrid(torch.arange(h), torch.arange(w)) + pos_y = pos_y.float().to(device) / float(h) + pos_x = pos_x.float().to(device) / float(w) + position_enc = torch.stack((pos_y, pos_x)) # shape [C, H, W] + + # unsqueeze and expand to match leading dimensions -> shape [..., C, H, W] + leading_shape = source_im.shape[:-3] + position_enc = position_enc[(None, ) * len(leading_shape)] + position_enc = position_enc.expand(*leading_shape, -1, -1, -1) + + # concat across channel dimension with input + source_im = torch.cat((source_im, position_enc), dim=-3) + + # make sure sample boundaries ensure crops are fully within the images + image_c, image_h, image_w = source_im.shape[-3:] + max_sample_h = image_h - crop_height + max_sample_w = image_w - crop_width + + # Sample crop locations for all tensor dimensions up to the last 3, which are [C, H, W]. + # Each gets @num_crops samples - typically this will just be the batch dimension (B), so + # we will sample [B, N] indices, but this supports having more than one leading dimension, + # or possibly no leading dimension. + # + # Trick: sample in [0, 1) with rand, then re-scale to [0, M) and convert to long to get sampled ints + crop_inds_h = (max_sample_h * torch.rand(*source_im.shape[:-3], num_crops).to(device)).long() + crop_inds_w = (max_sample_w * torch.rand(*source_im.shape[:-3], num_crops).to(device)).long() + crop_inds = torch.cat((crop_inds_h.unsqueeze(-1), crop_inds_w.unsqueeze(-1)), dim=-1) # shape [..., N, 2] + + crops = crop_image_from_indices( + images=source_im, + crop_indices=crop_inds, + crop_height=crop_height, + crop_width=crop_width, + ) + + return crops, crop_inds diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/vision/model_getter.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/vision/model_getter.py new file mode 100644 index 00000000..69972420 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/vision/model_getter.py @@ -0,0 +1,36 @@ +import torch +import torchvision + + +def get_resnet(name, weights=None, **kwargs): + """ + name: resnet18, resnet34, resnet50 + weights: "IMAGENET1K_V1", "r3m" + """ + # load r3m weights + if (weights == "r3m") or (weights == "R3M"): + return get_r3m(name=name, **kwargs) + + func = getattr(torchvision.models, name) + resnet = func(weights=weights, **kwargs) + resnet.fc = torch.nn.Identity() + # resnet_new = torch.nn.Sequential( + # resnet, + # torch.nn.Linear(512, 128) + # ) + # return resnet_new + return resnet + + +def get_r3m(name, **kwargs): + """ + name: resnet18, resnet34, resnet50 + """ + import r3m + + r3m.device = "cpu" + model = r3m.load_r3m(name) + r3m_model = model.module + resnet_model = r3m_model.convnet + resnet_model = resnet_model.to("cpu") + return resnet_model diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/vision/multi_image_obs_encoder.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/vision/multi_image_obs_encoder.py new file mode 100644 index 00000000..c7e77ac3 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/model/vision/multi_image_obs_encoder.py @@ -0,0 +1,191 @@ +from typing import Dict, Tuple, Union +import copy +import torch +import torch.nn as nn +import torchvision +from diffusion_policy.model.vision.crop_randomizer import CropRandomizer +from diffusion_policy.model.common.module_attr_mixin import ModuleAttrMixin +from diffusion_policy.common.pytorch_util import dict_apply, replace_submodules + + +class MultiImageObsEncoder(ModuleAttrMixin): + + def __init__( + self, + shape_meta: dict, + rgb_model: Union[nn.Module, Dict[str, nn.Module]], + resize_shape: Union[Tuple[int, int], Dict[str, tuple], None] = None, + crop_shape: Union[Tuple[int, int], Dict[str, tuple], None] = None, + random_crop: bool = True, + # replace BatchNorm with GroupNorm + use_group_norm: bool = False, + # use single rgb model for all rgb inputs + share_rgb_model: bool = False, + # renormalize rgb input with imagenet normalization + # assuming input in [0,1] + imagenet_norm: bool = False, + ): + """ + Assumes rgb input: B,C,H,W + Assumes low_dim input: B,D + """ + super().__init__() + + rgb_keys = list() + low_dim_keys = list() + key_model_map = nn.ModuleDict() + key_transform_map = nn.ModuleDict() + key_shape_map = dict() + + # handle sharing vision backbone + if share_rgb_model: + assert isinstance(rgb_model, nn.Module) + key_model_map["rgb"] = rgb_model + + obs_shape_meta = shape_meta["obs"] + for key, attr in obs_shape_meta.items(): + shape = tuple(attr["shape"]) + type = attr.get("type", "low_dim") + key_shape_map[key] = shape + if type == "rgb": + rgb_keys.append(key) + # configure model for this key + this_model = None + if not share_rgb_model: + if isinstance(rgb_model, dict): + # have provided model for each key + this_model = rgb_model[key] + else: + assert isinstance(rgb_model, nn.Module) + # have a copy of the rgb model + this_model = copy.deepcopy(rgb_model) + + if this_model is not None: + if use_group_norm: + this_model = replace_submodules( + root_module=this_model, + predicate=lambda x: isinstance(x, nn.BatchNorm2d), + func=lambda x: nn.GroupNorm( + num_groups=x.num_features // 16, + num_channels=x.num_features, + ), + ) + key_model_map[key] = this_model + + # configure resize + input_shape = shape + this_resizer = nn.Identity() + if resize_shape is not None: + if isinstance(resize_shape, dict): + h, w = resize_shape[key] + else: + h, w = resize_shape + this_resizer = torchvision.transforms.Resize(size=(h, w)) + input_shape = (shape[0], h, w) + + # configure randomizer + this_randomizer = nn.Identity() + if crop_shape is not None: + if isinstance(crop_shape, dict): + h, w = crop_shape[key] + else: + h, w = crop_shape + if random_crop: + this_randomizer = CropRandomizer( + input_shape=input_shape, + crop_height=h, + crop_width=w, + num_crops=1, + pos_enc=False, + ) + else: + this_normalizer = torchvision.transforms.CenterCrop(size=(h, w)) + # configure normalizer + this_normalizer = nn.Identity() + if imagenet_norm: + this_normalizer = torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) + + this_transform = nn.Sequential(this_resizer, this_randomizer, this_normalizer) + key_transform_map[key] = this_transform + elif type == "low_dim": + low_dim_keys.append(key) + else: + raise RuntimeError(f"Unsupported obs type: {type}") + rgb_keys = sorted(rgb_keys) + low_dim_keys = sorted(low_dim_keys) + + self.shape_meta = shape_meta + self.key_model_map = key_model_map + self.key_transform_map = key_transform_map + self.share_rgb_model = share_rgb_model + self.rgb_keys = rgb_keys + self.low_dim_keys = low_dim_keys + self.key_shape_map = key_shape_map + + def forward(self, obs_dict): + batch_size = None + features = list() + # process rgb input + if self.share_rgb_model: + # pass all rgb obs to rgb model + imgs = list() + for key in self.rgb_keys: + img = obs_dict[key] + if batch_size is None: + batch_size = img.shape[0] + else: + assert batch_size == img.shape[0] + assert img.shape[1:] == self.key_shape_map[key] + img = self.key_transform_map[key](img) + imgs.append(img) + # (N*B,C,H,W) + imgs = torch.cat(imgs, dim=0) + # (N*B,D) + feature = self.key_model_map["rgb"](imgs) + # (N,B,D) + feature = feature.reshape(-1, batch_size, *feature.shape[1:]) + # (B,N,D) + feature = torch.moveaxis(feature, 0, 1) + # (B,N*D) + feature = feature.reshape(batch_size, -1) + features.append(feature) + else: + # run each rgb obs to independent models + for key in self.rgb_keys: + img = obs_dict[key] + if batch_size is None: + batch_size = img.shape[0] + else: + assert batch_size == img.shape[0] + assert img.shape[1:] == self.key_shape_map[key] + img = self.key_transform_map[key](img) + feature = self.key_model_map[key](img) + features.append(feature) + + # process lowdim input + for key in self.low_dim_keys: + data = obs_dict[key] + if batch_size is None: + batch_size = data.shape[0] + else: + assert batch_size == data.shape[0] + assert data.shape[1:] == self.key_shape_map[key] + features.append(data) + + # concatenate all features + result = torch.cat(features, dim=-1) + return result + + @torch.no_grad() + def output_shape(self): + example_obs_dict = dict() + obs_shape_meta = self.shape_meta["obs"] + batch_size = 1 + for key, attr in obs_shape_meta.items(): + shape = tuple(attr["shape"]) + this_obs = torch.zeros((batch_size, ) + shape, dtype=self.dtype, device=self.device) + example_obs_dict[key] = this_obs + example_output = self.forward(example_obs_dict) + output_shape = example_output.shape[1:] + return output_shape diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/policy/base_image_policy.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/policy/base_image_policy.py new file mode 100644 index 00000000..3a926a35 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/policy/base_image_policy.py @@ -0,0 +1,26 @@ +from typing import Dict +import torch +import torch.nn as nn +from diffusion_policy.model.common.module_attr_mixin import ModuleAttrMixin +from diffusion_policy.model.common.normalizer import LinearNormalizer + + +class BaseImagePolicy(ModuleAttrMixin): + # init accepts keyword argument shape_meta, see config/task/*_image.yaml + + def predict_action(self, obs_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """ + obs_dict: + str: B,To,* + return: B,Ta,Da + """ + raise NotImplementedError() + + # reset state for stateful policies + def reset(self): + pass + + # ========== training =========== + # no standard training interface except setting normalizer + def set_normalizer(self, normalizer: LinearNormalizer): + raise NotImplementedError() diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/policy/diffusion_unet_image_policy.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/policy/diffusion_unet_image_policy.py new file mode 100644 index 00000000..c93c302f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/policy/diffusion_unet_image_policy.py @@ -0,0 +1,258 @@ +from typing import Dict +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange, reduce +from diffusers.schedulers.scheduling_ddpm import DDPMScheduler + +from diffusion_policy.model.common.normalizer import LinearNormalizer +from diffusion_policy.policy.base_image_policy import BaseImagePolicy +from diffusion_policy.model.diffusion.conditional_unet1d import ConditionalUnet1D +from diffusion_policy.model.diffusion.mask_generator import LowdimMaskGenerator +from diffusion_policy.model.vision.multi_image_obs_encoder import MultiImageObsEncoder +from diffusion_policy.common.pytorch_util import dict_apply + + +class DiffusionUnetImagePolicy(BaseImagePolicy): + + def __init__( + self, + shape_meta: dict, + noise_scheduler: DDPMScheduler, + obs_encoder: MultiImageObsEncoder, + horizon, + n_action_steps, + n_obs_steps, + num_inference_steps=None, + obs_as_global_cond=True, + diffusion_step_embed_dim=256, + down_dims=(256, 512, 1024), + kernel_size=5, + n_groups=8, + cond_predict_scale=True, + # parameters passed to step + **kwargs, + ): + super().__init__() + + # parse shapes + action_shape = shape_meta["action"]["shape"] + assert len(action_shape) == 1 + action_dim = action_shape[0] + # get feature dim + obs_feature_dim = obs_encoder.output_shape()[0] + + # create diffusion model + input_dim = action_dim + obs_feature_dim + global_cond_dim = None + if obs_as_global_cond: + input_dim = action_dim + global_cond_dim = obs_feature_dim * n_obs_steps + + model = ConditionalUnet1D( + input_dim=input_dim, + local_cond_dim=None, + global_cond_dim=global_cond_dim, + diffusion_step_embed_dim=diffusion_step_embed_dim, + down_dims=down_dims, + kernel_size=kernel_size, + n_groups=n_groups, + cond_predict_scale=cond_predict_scale, + ) + + self.obs_encoder = obs_encoder + self.model = model + self.noise_scheduler = noise_scheduler + self.mask_generator = LowdimMaskGenerator( + action_dim=action_dim, + obs_dim=0 if obs_as_global_cond else obs_feature_dim, + max_n_obs_steps=n_obs_steps, + fix_obs_steps=True, + action_visible=False, + ) + self.normalizer = LinearNormalizer() + self.horizon = horizon + self.obs_feature_dim = obs_feature_dim + self.action_dim = action_dim + self.n_action_steps = n_action_steps + self.n_obs_steps = n_obs_steps + self.obs_as_global_cond = obs_as_global_cond + self.kwargs = kwargs + + if num_inference_steps is None: + num_inference_steps = noise_scheduler.config.num_train_timesteps + self.num_inference_steps = num_inference_steps + + # ========= inference ============ + def conditional_sample( + self, + condition_data, + condition_mask, + local_cond=None, + global_cond=None, + generator=None, + # keyword arguments to scheduler.step + **kwargs, + ): + model = self.model + scheduler = self.noise_scheduler + + trajectory = torch.randn( + size=condition_data.shape, + dtype=condition_data.dtype, + device=condition_data.device, + generator=generator, + ) + + # set step values + scheduler.set_timesteps(self.num_inference_steps) + + for t in scheduler.timesteps: + # 1. apply conditioning + trajectory[condition_mask] = condition_data[condition_mask] + + # 2. predict model output + model_output = model(trajectory, t, local_cond=local_cond, global_cond=global_cond) + + # 3. compute previous image: x_t -> x_t-1 + trajectory = scheduler.step(model_output, t, trajectory, generator=generator, **kwargs).prev_sample + + # finally make sure conditioning is enforced + trajectory[condition_mask] = condition_data[condition_mask] + + return trajectory + + def predict_action(self, obs_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """ + obs_dict: must include "obs" key + result: must include "action" key + """ + assert "past_action" not in obs_dict # not implemented yet + # normalize input + nobs = self.normalizer.normalize(obs_dict) + value = next(iter(nobs.values())) + B, To = value.shape[:2] + T = self.horizon + Da = self.action_dim + Do = self.obs_feature_dim + To = self.n_obs_steps + + # build input + device = self.device + dtype = self.dtype + + # handle different ways of passing observation + local_cond = None + global_cond = None + if self.obs_as_global_cond: + # condition through global feature + this_nobs = dict_apply(nobs, lambda x: x[:, :To, ...].reshape(-1, *x.shape[2:])) + nobs_features = self.obs_encoder(this_nobs) + # reshape back to B, Do + global_cond = nobs_features.reshape(B, -1) + # empty data for action + cond_data = torch.zeros(size=(B, T, Da), device=device, dtype=dtype) + cond_mask = torch.zeros_like(cond_data, dtype=torch.bool) + else: + # condition through impainting + this_nobs = dict_apply(nobs, lambda x: x[:, :To, ...].reshape(-1, *x.shape[2:])) + nobs_features = self.obs_encoder(this_nobs) + # reshape back to B, T, Do + nobs_features = nobs_features.reshape(B, To, -1) + cond_data = torch.zeros(size=(B, T, Da + Do), device=device, dtype=dtype) + cond_mask = torch.zeros_like(cond_data, dtype=torch.bool) + cond_data[:, :To, Da:] = nobs_features + cond_mask[:, :To, Da:] = True + + # run sampling + nsample = self.conditional_sample( + cond_data, + cond_mask, + local_cond=local_cond, + global_cond=global_cond, + **self.kwargs, + ) + + # unnormalize prediction + naction_pred = nsample[..., :Da] + action_pred = self.normalizer["action"].unnormalize(naction_pred) + + # get action + start = To - 1 + end = start + self.n_action_steps + action = action_pred[:, start:end] + + result = {"action": action, "action_pred": action_pred} + return result + + # ========= training ============ + def set_normalizer(self, normalizer: LinearNormalizer): + self.normalizer.load_state_dict(normalizer.state_dict()) + + def compute_loss(self, batch): + # normalize input + assert "valid_mask" not in batch + nobs = self.normalizer.normalize(batch["obs"]) + nactions = self.normalizer["action"].normalize(batch["action"]) + batch_size = nactions.shape[0] + horizon = nactions.shape[1] + + # handle different ways of passing observation + local_cond = None + global_cond = None + trajectory = nactions + cond_data = trajectory + if self.obs_as_global_cond: + # reshape B, T, ... to B*T + this_nobs = dict_apply(nobs, lambda x: x[:, :self.n_obs_steps, ...].reshape(-1, *x.shape[2:])) + nobs_features = self.obs_encoder(this_nobs) + # reshape back to B, Do + global_cond = nobs_features.reshape(batch_size, -1) + else: + # reshape B, T, ... to B*T + this_nobs = dict_apply(nobs, lambda x: x.reshape(-1, *x.shape[2:])) + nobs_features = self.obs_encoder(this_nobs) + # reshape back to B, T, Do + nobs_features = nobs_features.reshape(batch_size, horizon, -1) + cond_data = torch.cat([nactions, nobs_features], dim=-1) + trajectory = cond_data.detach() + + # generate impainting mask + condition_mask = self.mask_generator(trajectory.shape) + + # Sample noise that we'll add to the images + noise = torch.randn(trajectory.shape, device=trajectory.device) + bsz = trajectory.shape[0] + # Sample a random timestep for each image + timesteps = torch.randint( + 0, + self.noise_scheduler.config.num_train_timesteps, + (bsz, ), + device=trajectory.device, + ).long() + # Add noise to the clean images according to the noise magnitude at each timestep + # (this is the forward diffusion process) + noisy_trajectory = self.noise_scheduler.add_noise(trajectory, noise, timesteps) + + # compute loss mask + loss_mask = ~condition_mask + + # apply conditioning + noisy_trajectory[condition_mask] = cond_data[condition_mask] + + # Predict the noise residual + pred = self.model(noisy_trajectory, timesteps, local_cond=local_cond, global_cond=global_cond) + + pred_type = self.noise_scheduler.config.prediction_type + if pred_type == "epsilon": + target = noise + elif pred_type == "sample": + target = trajectory + else: + raise ValueError(f"Unsupported prediction type {pred_type}") + + loss = F.mse_loss(pred, target, reduction="none") + loss = loss * loss_mask.type(loss.dtype) + loss = reduce(loss, "b ... -> b (...)", "mean") + loss = loss.mean() + return loss diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/shared_memory/shared_memory_queue.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/shared_memory/shared_memory_queue.py new file mode 100644 index 00000000..39a099d4 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/shared_memory/shared_memory_queue.py @@ -0,0 +1,184 @@ +from typing import Dict, List, Union +import numbers +from queue import Empty, Full +from multiprocessing.managers import SharedMemoryManager +import numpy as np +from diffusion_policy.shared_memory.shared_memory_util import ( + ArraySpec, + SharedAtomicCounter, +) +from diffusion_policy.shared_memory.shared_ndarray import SharedNDArray + + +class SharedMemoryQueue: + """ + A Lock-Free FIFO Shared Memory Data Structure. + Stores a sequence of dict of numpy arrays. + """ + + def __init__( + self, + shm_manager: SharedMemoryManager, + array_specs: List[ArraySpec], + buffer_size: int, + ): + + # create atomic counter + write_counter = SharedAtomicCounter(shm_manager) + read_counter = SharedAtomicCounter(shm_manager) + + # allocate shared memory + shared_arrays = dict() + for spec in array_specs: + key = spec.name + assert key not in shared_arrays + array = SharedNDArray.create_from_shape( + mem_mgr=shm_manager, + shape=(buffer_size, ) + tuple(spec.shape), + dtype=spec.dtype, + ) + shared_arrays[key] = array + + self.buffer_size = buffer_size + self.array_specs = array_specs + self.write_counter = write_counter + self.read_counter = read_counter + self.shared_arrays = shared_arrays + + @classmethod + def create_from_examples( + cls, + shm_manager: SharedMemoryManager, + examples: Dict[str, Union[np.ndarray, numbers.Number]], + buffer_size: int, + ): + specs = list() + for key, value in examples.items(): + shape = None + dtype = None + if isinstance(value, np.ndarray): + shape = value.shape + dtype = value.dtype + assert dtype != np.dtype("O") + elif isinstance(value, numbers.Number): + shape = tuple() + dtype = np.dtype(type(value)) + else: + raise TypeError(f"Unsupported type {type(value)}") + + spec = ArraySpec(name=key, shape=shape, dtype=dtype) + specs.append(spec) + + obj = cls(shm_manager=shm_manager, array_specs=specs, buffer_size=buffer_size) + return obj + + def qsize(self): + read_count = self.read_counter.load() + write_count = self.write_counter.load() + n_data = write_count - read_count + return n_data + + def empty(self): + n_data = self.qsize() + return n_data <= 0 + + def clear(self): + self.read_counter.store(self.write_counter.load()) + + def put(self, data: Dict[str, Union[np.ndarray, numbers.Number]]): + read_count = self.read_counter.load() + write_count = self.write_counter.load() + n_data = write_count - read_count + if n_data >= self.buffer_size: + raise Full() + + next_idx = write_count % self.buffer_size + + # write to shared memory + for key, value in data.items(): + arr: np.ndarray + arr = self.shared_arrays[key].get() + if isinstance(value, np.ndarray): + arr[next_idx] = value + else: + arr[next_idx] = np.array(value, dtype=arr.dtype) + + # update idx + self.write_counter.add(1) + + def get(self, out=None) -> Dict[str, np.ndarray]: + write_count = self.write_counter.load() + read_count = self.read_counter.load() + n_data = write_count - read_count + if n_data <= 0: + raise Empty() + + if out is None: + out = self._allocate_empty() + + next_idx = read_count % self.buffer_size + for key, value in self.shared_arrays.items(): + arr = value.get() + np.copyto(out[key], arr[next_idx]) + + # update idx + self.read_counter.add(1) + return out + + def get_k(self, k, out=None) -> Dict[str, np.ndarray]: + write_count = self.write_counter.load() + read_count = self.read_counter.load() + n_data = write_count - read_count + if n_data <= 0: + raise Empty() + assert k <= n_data + + out = self._get_k_impl(k, read_count, out=out) + self.read_counter.add(k) + return out + + def get_all(self, out=None) -> Dict[str, np.ndarray]: + write_count = self.write_counter.load() + read_count = self.read_counter.load() + n_data = write_count - read_count + if n_data <= 0: + raise Empty() + + out = self._get_k_impl(n_data, read_count, out=out) + self.read_counter.add(n_data) + return out + + def _get_k_impl(self, k, read_count, out=None) -> Dict[str, np.ndarray]: + if out is None: + out = self._allocate_empty(k) + + curr_idx = read_count % self.buffer_size + for key, value in self.shared_arrays.items(): + arr = value.get() + target = out[key] + + start = curr_idx + end = min(start + k, self.buffer_size) + target_start = 0 + target_end = end - start + target[target_start:target_end] = arr[start:end] + + remainder = k - (end - start) + if remainder > 0: + # wrap around + start = 0 + end = start + remainder + target_start = target_end + target_end = k + target[target_start:target_end] = arr[start:end] + + return out + + def _allocate_empty(self, k=None): + result = dict() + for spec in self.array_specs: + shape = spec.shape + if k is not None: + shape = (k, ) + shape + result[spec.name] = np.empty(shape=shape, dtype=spec.dtype) + return result diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/shared_memory/shared_memory_ring_buffer.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/shared_memory/shared_memory_ring_buffer.py new file mode 100644 index 00000000..d918eb79 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/shared_memory/shared_memory_ring_buffer.py @@ -0,0 +1,213 @@ +from typing import Dict, List, Union + +from queue import Empty +import numbers +import time +from multiprocessing.managers import SharedMemoryManager +import numpy as np + +from diffusion_policy.shared_memory.shared_ndarray import SharedNDArray +from diffusion_policy.shared_memory.shared_memory_util import ( + ArraySpec, + SharedAtomicCounter, +) + + +class SharedMemoryRingBuffer: + """ + A Lock-Free FILO Shared Memory Data Structure. + Stores a sequence of dict of numpy arrays. + """ + + def __init__( + self, + shm_manager: SharedMemoryManager, + array_specs: List[ArraySpec], + get_max_k: int, + get_time_budget: float, + put_desired_frequency: float, + safety_margin: float = 1.5, + ): + """ + shm_manager: Manages the life cycle of share memories + across processes. Remember to run .start() before passing. + array_specs: Name, shape and type of arrays for a single time step. + get_max_k: The maxmum number of items can be queried at once. + get_time_budget: The maxmum amount of time spent copying data from + shared memory to local memory. Increase this number for larger arrays. + put_desired_frequency: The maximum frequency that .put() can be called. + This influces the buffer size. + """ + + # create atomic counter + counter = SharedAtomicCounter(shm_manager) + + # compute buffer size + # At any given moment, the past get_max_k items should never + # be touched (to be read freely). Assuming the reading is reading + # these k items, which takes maximum of get_time_budget seconds, + # we need enough empty slots to make sure put_desired_frequency Hz + # of put can be sustaied. + buffer_size = (int(np.ceil(put_desired_frequency * get_time_budget * safety_margin)) + get_max_k) + + # allocate shared memory + shared_arrays = dict() + for spec in array_specs: + key = spec.name + assert key not in shared_arrays + array = SharedNDArray.create_from_shape( + mem_mgr=shm_manager, + shape=(buffer_size, ) + tuple(spec.shape), + dtype=spec.dtype, + ) + shared_arrays[key] = array + + # allocate timestamp array + timestamp_array = SharedNDArray.create_from_shape(mem_mgr=shm_manager, shape=(buffer_size, ), dtype=np.float64) + timestamp_array.get()[:] = -np.inf + + self.buffer_size = buffer_size + self.array_specs = array_specs + self.counter = counter + self.shared_arrays = shared_arrays + self.timestamp_array = timestamp_array + self.get_time_budget = get_time_budget + self.get_max_k = get_max_k + self.put_desired_frequency = put_desired_frequency + + @property + def count(self): + return self.counter.load() + + @classmethod + def create_from_examples( + cls, + shm_manager: SharedMemoryManager, + examples: Dict[str, Union[np.ndarray, numbers.Number]], + get_max_k: int = 32, + get_time_budget: float = 0.01, + put_desired_frequency: float = 60, + ): + specs = list() + for key, value in examples.items(): + shape = None + dtype = None + if isinstance(value, np.ndarray): + shape = value.shape + dtype = value.dtype + assert dtype != np.dtype("O") + elif isinstance(value, numbers.Number): + shape = tuple() + dtype = np.dtype(type(value)) + else: + raise TypeError(f"Unsupported type {type(value)}") + + spec = ArraySpec(name=key, shape=shape, dtype=dtype) + specs.append(spec) + + obj = cls( + shm_manager=shm_manager, + array_specs=specs, + get_max_k=get_max_k, + get_time_budget=get_time_budget, + put_desired_frequency=put_desired_frequency, + ) + return obj + + def clear(self): + self.counter.store(0) + + def put(self, data: Dict[str, Union[np.ndarray, numbers.Number]], wait: bool = True): + count = self.counter.load() + next_idx = count % self.buffer_size + # Make sure the next self.get_max_k elements in the ring buffer have at least + # self.get_time_budget seconds untouched after written, so that + # get_last_k can safely read k elements from any count location. + # Sanity check: when get_max_k == 1, the element pointed by next_idx + # should be rewritten at minimum self.get_time_budget seconds later. + timestamp_lookahead_idx = (next_idx + self.get_max_k - 1) % self.buffer_size + old_timestamp = self.timestamp_array.get()[timestamp_lookahead_idx] + t = time.monotonic() + if (t - old_timestamp) < self.get_time_budget: + deltat = t - old_timestamp + if wait: + # sleep the remaining time to be safe + time.sleep(self.get_time_budget - deltat) + else: + # throw an error + past_iters = self.buffer_size - self.get_max_k + hz = past_iters / deltat + raise TimeoutError("Put executed too fast {}items/{:.4f}s ~= {}Hz".format(past_iters, deltat, hz)) + + # write to shared memory + for key, value in data.items(): + arr: np.ndarray + arr = self.shared_arrays[key].get() + if isinstance(value, np.ndarray): + arr[next_idx] = value + else: + arr[next_idx] = np.array(value, dtype=arr.dtype) + + # update timestamp + self.timestamp_array.get()[next_idx] = time.monotonic() + self.counter.add(1) + + def _allocate_empty(self, k=None): + result = dict() + for spec in self.array_specs: + shape = spec.shape + if k is not None: + shape = (k, ) + shape + result[spec.name] = np.empty(shape=shape, dtype=spec.dtype) + return result + + def get(self, out=None) -> Dict[str, np.ndarray]: + if out is None: + out = self._allocate_empty() + start_time = time.monotonic() + count = self.counter.load() + curr_idx = (count - 1) % self.buffer_size + for key, value in self.shared_arrays.items(): + arr = value.get() + np.copyto(out[key], arr[curr_idx]) + end_time = time.monotonic() + dt = end_time - start_time + if dt > self.get_time_budget: + raise TimeoutError(f"Get time out {dt} vs {self.get_time_budget}") + return out + + def get_last_k(self, k: int, out=None) -> Dict[str, np.ndarray]: + assert k <= self.get_max_k + if out is None: + out = self._allocate_empty(k) + start_time = time.monotonic() + count = self.counter.load() + assert k <= count + curr_idx = (count - 1) % self.buffer_size + for key, value in self.shared_arrays.items(): + arr = value.get() + target = out[key] + + end = curr_idx + 1 + start = max(0, end - k) + target_end = k + target_start = target_end - (end - start) + target[target_start:target_end] = arr[start:end] + + remainder = k - (end - start) + if remainder > 0: + # wrap around + end = self.buffer_size + start = end - remainder + target_start = 0 + target_end = end - start + target[target_start:target_end] = arr[start:end] + end_time = time.monotonic() + dt = end_time - start_time + if dt > self.get_time_budget: + raise TimeoutError(f"Get time out {dt} vs {self.get_time_budget}") + return out + + def get_all(self) -> Dict[str, np.ndarray]: + k = min(self.count, self.get_max_k) + return self.get_last_k(k=k) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/shared_memory/shared_memory_util.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/shared_memory/shared_memory_util.py new file mode 100644 index 00000000..2396208a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/shared_memory/shared_memory_util.py @@ -0,0 +1,38 @@ +from typing import Tuple +from dataclasses import dataclass +import numpy as np +from multiprocessing.managers import SharedMemoryManager +from atomics import atomicview, MemoryOrder, UINT + + +@dataclass +class ArraySpec: + name: str + shape: Tuple[int] + dtype: np.dtype + + +class SharedAtomicCounter: + + def __init__(self, shm_manager: SharedMemoryManager, size: int = 8): # 64bit int + shm = shm_manager.SharedMemory(size=size) + self.shm = shm + self.size = size + self.store(0) # initialize + + @property + def buf(self): + return self.shm.buf[:self.size] + + def load(self) -> int: + with atomicview(buffer=self.buf, atype=UINT) as a: + value = a.load(order=MemoryOrder.ACQUIRE) + return value + + def store(self, value: int): + with atomicview(buffer=self.buf, atype=UINT) as a: + a.store(value, order=MemoryOrder.RELEASE) + + def add(self, value: int): + with atomicview(buffer=self.buf, atype=UINT) as a: + a.add(value, order=MemoryOrder.ACQ_REL) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/shared_memory/shared_ndarray.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/shared_memory/shared_ndarray.py new file mode 100644 index 00000000..d027bb84 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/shared_memory/shared_ndarray.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import multiprocessing +import multiprocessing.synchronize +from multiprocessing.managers import SharedMemoryManager +from multiprocessing.shared_memory import SharedMemory +from typing import Any, TYPE_CHECKING, Generic, Optional, Tuple, TypeVar, Union + +import numpy as np +import numpy.typing as npt +from diffusion_policy.common.nested_dict_util import nested_dict_check, nested_dict_map + +SharedMemoryLike = Union[str, SharedMemory] # shared memory or name of shared memory +SharedT = TypeVar("SharedT", bound=np.generic) + + +class SharedNDArray(Generic[SharedT]): + """Class to keep track of and retrieve the data in a shared array + Attributes + ---------- + shm + SharedMemory object containing the data of the array + shape + Shape of the NumPy array + dtype + Type of the NumPy array. Anything that may be passed to the `dtype=` argument in `np.ndarray`. + lock + (Optional) multiprocessing.Lock to manage access to the SharedNDArray. This is only created if + lock=True is passed to the constructor, otherwise it is set to `None`. + A SharedNDArray object may be created either directly with a preallocated shared memory object plus the + dtype and shape of the numpy array it represents: + >>> from multiprocessing.shared_memory import SharedMemory + >>> import numpy as np + >>> from shared_ndarray2 import SharedNDArray + >>> x = np.array([1, 2, 3]) + >>> shm = SharedMemory(name="x", create=True, size=x.nbytes) + >>> arr = SharedNDArray(shm, x.shape, x.dtype) + >>> arr[:] = x[:] # copy x into the array + >>> print(arr[:]) + [1 2 3] + >>> shm.close() + >>> shm.unlink() + Or using a SharedMemoryManager either from an existing array or from arbitrary shape and nbytes: + >>> from multiprocessing.managers import SharedMemoryManager + >>> mem_mgr = SharedMemoryManager() + >>> mem_mgr.start() # Better yet, use SharedMemoryManager context manager + >>> arr = SharedNDArray.from_shape(mem_mgr, x.shape, x.dtype) + >>> arr[:] = x[:] # copy x into the array + >>> print(arr[:]) + [1 2 3] + >>> # -or in one step- + >>> arr = SharedNDArray.from_array(mem_mgr, x) + >>> print(arr[:]) + [1 2 3] + `SharedNDArray` does not subclass numpy.ndarray but rather generates an ndarray on-the-fly in get(), + which is used in __getitem__ and __setitem__. Thus to access the data and/or use any ndarray methods + get() or __getitem__ or __setitem__ must be used + >>> arr.max() # ERROR: SharedNDArray has no `max` method. + Traceback (most recent call last): + .... + AttributeError: SharedNDArray object has no attribute 'max'. To access NumPy ndarray object use .get() method. + >>> arr.get().max() # (or arr[:].max()) OK: This gets an ndarray on which we can operate + 3 + >>> y = np.zeros(3) + >>> y[:] = arr # ERROR: Cannot broadcast-assign a SharedNDArray to ndarray `y` + Traceback (most recent call last): + ... + ValueError: setting an array element with a sequence. + >>> y[:] = arr[:] # OK: This gets an ndarray that can be copied element-wise to `y` + >>> mem_mgr.shutdown() + """ + + shm: SharedMemory + # shape: Tuple[int, ...] # is a property + dtype: np.dtype + lock: Optional[multiprocessing.synchronize.Lock] + + def __init__(self, shm: SharedMemoryLike, shape: Tuple[int, ...], dtype: npt.DTypeLike): + """Initialize a SharedNDArray object from existing shared memory, object shape, and dtype. + To initialize a SharedNDArray object from a memory manager and data or shape, use the `from_array() + or `from_shape()` classmethods. + Parameters + ---------- + shm + `multiprocessing.shared_memory.SharedMemory` object or name for connecting to an existing block + of shared memory (using SharedMemory constructor) + shape + Shape of the NumPy array to be represented in the shared memory + dtype + Data type for the NumPy array to be represented in shared memory. Any valid argument for + `np.dtype` may be used as it will be converted to an actual `dtype` object. + lock : bool, optional + If True, create a multiprocessing.Lock object accessible with the `.lock` attribute, by default + False. If passing the `SharedNDArray` as an argument to a `multiprocessing.Pool` function this + should not be used -- see this comment to a Stack Overflow question about `multiprocessing.Lock`: + https://stackoverflow.com/questions/25557686/python-sharing-a-lock-between-processes#comment72803059_25558333 + Raises + ------ + ValueError + The SharedMemory size (number of bytes) does not match the product of the shape and dtype + itemsize. + """ + if isinstance(shm, str): + shm = SharedMemory(name=shm, create=False) + dtype = np.dtype(dtype) # Try to convert to dtype + assert shm.size >= (dtype.itemsize * np.prod(shape)) + self.shm = shm + self.dtype = dtype + self._shape: Tuple[int, ...] = shape + + def __repr__(self): + # Like numpy's ndarray repr + cls_name = self.__class__.__name__ + nspaces = len(cls_name) + 1 + array_repr = str(self.get()) + array_repr = array_repr.replace("\n", "\n" + " " * nspaces) + return f"{cls_name}({array_repr}, dtype={self.dtype})" + + @classmethod + def create_from_array(cls, mem_mgr: SharedMemoryManager, arr: npt.NDArray[SharedT]) -> SharedNDArray[SharedT]: + """Create a SharedNDArray from a SharedMemoryManager and an existing numpy array. + Parameters + ---------- + mem_mgr + Running `multiprocessing.managers.SharedMemoryManager` instance from which to create the + SharedMemory for the SharedNDArray + arr + NumPy `ndarray` object to copy into the created SharedNDArray upon initialization. + """ + # Simply use from_shape() to create the SharedNDArray and copy the data into it. + shared_arr = cls.create_from_shape(mem_mgr, arr.shape, arr.dtype) + shared_arr.get()[:] = arr[:] + return shared_arr + + @classmethod + def create_from_shape(cls, mem_mgr: SharedMemoryManager, shape: Tuple, dtype: npt.DTypeLike) -> SharedNDArray: + """Create a SharedNDArray directly from a SharedMemoryManager + Parameters + ---------- + mem_mgr + SharedMemoryManager instance that has been started + shape + Shape of the array + dtype + Data type for the NumPy array to be represented in shared memory. Any valid argument for + `np.dtype` may be used as it will be converted to an actual `dtype` object. + """ + dtype = np.dtype(dtype) # Convert to dtype if possible + shm = mem_mgr.SharedMemory(np.prod(shape) * dtype.itemsize) + return cls(shm=shm, shape=shape, dtype=dtype) + + @property + def shape(self) -> Tuple[int, ...]: + return self._shape + + def get(self) -> npt.NDArray[SharedT]: + """Get a numpy array with access to the shared memory""" + return np.ndarray(self.shape, dtype=self.dtype, buffer=self.shm.buf) + + def __del__(self): + self.shm.close() diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/workspace/base_workspace.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/workspace/base_workspace.py new file mode 100644 index 00000000..d11c1579 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/workspace/base_workspace.py @@ -0,0 +1,138 @@ +from typing import Optional +import os +import pathlib +import hydra +import copy +from hydra.core.hydra_config import HydraConfig +from omegaconf import OmegaConf +import dill +import torch +import threading + + +class BaseWorkspace: + include_keys = tuple() + exclude_keys = tuple() + + def __init__(self, cfg: OmegaConf, output_dir: Optional[str] = None): + self.cfg = cfg + self._output_dir = output_dir + self._saving_thread = None + + @property + def output_dir(self): + output_dir = self._output_dir + if output_dir is None: + output_dir = HydraConfig.get().runtime.output_dir + return output_dir + + def run(self): + """ + Create any resource shouldn't be serialized as local variables + """ + pass + + def save_checkpoint( + self, + path=None, + tag="latest", + exclude_keys=None, + include_keys=None, + use_thread=True, + ): + if path is None: + path = pathlib.Path(self.output_dir).joinpath("checkpoints", f"{tag}.ckpt") + else: + path = pathlib.Path(path) + if exclude_keys is None: + exclude_keys = tuple(self.exclude_keys) + if include_keys is None: + include_keys = tuple(self.include_keys) + ("_output_dir", ) + + path.parent.mkdir(parents=True, exist_ok=True) + payload = {"cfg": self.cfg, "state_dicts": dict(), "pickles": dict()} + + for key, value in self.__dict__.items(): + if hasattr(value, "state_dict") and hasattr(value, "load_state_dict"): + # modules, optimizers and samplers etc + if key not in exclude_keys: + if use_thread: + payload["state_dicts"][key] = _copy_to_cpu(value.state_dict()) + else: + payload["state_dicts"][key] = value.state_dict() + elif key in include_keys: + payload["pickles"][key] = dill.dumps(value) + if use_thread: + self._saving_thread = threading.Thread( + target=lambda: torch.save(payload, path.open("wb"), pickle_module=dill)) + self._saving_thread.start() + else: + torch.save(payload, path.open("wb"), pickle_module=dill) + return str(path.absolute()) + + def get_checkpoint_path(self, tag="latest"): + return pathlib.Path(self.output_dir).joinpath("checkpoints", f"{tag}.ckpt") + + def load_payload(self, payload, exclude_keys=None, include_keys=None, **kwargs): + if exclude_keys is None: + exclude_keys = tuple() + if include_keys is None: + include_keys = payload["pickles"].keys() + + for key, value in payload["state_dicts"].items(): + if key not in exclude_keys: + self.__dict__[key].load_state_dict(value, **kwargs) + for key in include_keys: + if key in payload["pickles"]: + self.__dict__[key] = dill.loads(payload["pickles"][key]) + + def load_checkpoint(self, path=None, tag="latest", exclude_keys=None, include_keys=None, **kwargs): + if path is None: + path = self.get_checkpoint_path(tag=tag) + else: + path = pathlib.Path(path) + payload = torch.load(path.open("rb"), pickle_module=dill, **kwargs) + self.load_payload(payload, exclude_keys=exclude_keys, include_keys=include_keys) + return payload + + @classmethod + def create_from_checkpoint(cls, path, exclude_keys=None, include_keys=None, **kwargs): + payload = torch.load(open(path, "rb"), pickle_module=dill) + instance = cls(payload["cfg"]) + instance.load_payload( + payload=payload, + exclude_keys=exclude_keys, + include_keys=include_keys, + **kwargs, + ) + return instance + + def save_snapshot(self, tag="latest"): + """ + Quick loading and saving for reserach, saves full state of the workspace. + + However, loading a snapshot assumes the code stays exactly the same. + Use save_checkpoint for long-term storage. + """ + path = pathlib.Path(self.output_dir).joinpath("snapshots", f"{tag}.pkl") + path.parent.mkdir(parents=False, exist_ok=True) + torch.save(self, path.open("wb"), pickle_module=dill) + return str(path.absolute()) + + @classmethod + def create_from_snapshot(cls, path): + return torch.load(open(path, "rb"), pickle_module=dill) + + +def _copy_to_cpu(x): + if isinstance(x, torch.Tensor): + return x.detach().to("cpu") + elif isinstance(x, dict): + result = dict() + for k, v in x.items(): + result[k] = _copy_to_cpu(v) + return result + elif isinstance(x, list): + return [_copy_to_cpu(k) for k in x] + else: + return copy.deepcopy(x) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/workspace/robotworkspace.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/workspace/robotworkspace.py new file mode 100644 index 00000000..8575a7e0 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/diffusion_policy/workspace/robotworkspace.py @@ -0,0 +1,348 @@ +if __name__ == "__main__": + import sys + import os + import pathlib + + ROOT_DIR = str(pathlib.Path(__file__).parent.parent.parent) + sys.path.append(ROOT_DIR) + os.chdir(ROOT_DIR) + +import os +import hydra +import torch +from omegaconf import OmegaConf +import pathlib +from torch.utils.data import DataLoader +import copy + +import tqdm, random +import numpy as np +from diffusion_policy.workspace.base_workspace import BaseWorkspace +from diffusion_policy.policy.diffusion_unet_image_policy import DiffusionUnetImagePolicy +from diffusion_policy.dataset.base_dataset import BaseImageDataset +from diffusion_policy.common.checkpoint_util import TopKCheckpointManager +from diffusion_policy.common.json_logger import JsonLogger +from diffusion_policy.common.pytorch_util import dict_apply, optimizer_to +from diffusion_policy.model.diffusion.ema_model import EMAModel +from diffusion_policy.model.common.lr_scheduler import get_scheduler + +OmegaConf.register_new_resolver("eval", eval, replace=True) + + +class RobotWorkspace(BaseWorkspace): + include_keys = ["global_step", "epoch"] + + def __init__(self, cfg: OmegaConf, output_dir=None): + super().__init__(cfg, output_dir=output_dir) + + # set seed + seed = cfg.training.seed + torch.manual_seed(seed) + np.random.seed(seed) + random.seed(seed) + + # configure model + self.model: DiffusionUnetImagePolicy = hydra.utils.instantiate(cfg.policy) + + self.ema_model: DiffusionUnetImagePolicy = None + if cfg.training.use_ema: + self.ema_model = copy.deepcopy(self.model) + + # configure training state + self.optimizer = hydra.utils.instantiate(cfg.optimizer, params=self.model.parameters()) + + # configure training state + self.global_step = 0 + self.epoch = 0 + + def run(self): + cfg = copy.deepcopy(self.cfg) + seed = cfg.training.seed + head_camera_type = cfg.head_camera_type + + # resume training + if cfg.training.resume: + lastest_ckpt_path = self.get_checkpoint_path() + if lastest_ckpt_path.is_file(): + print(f"Resuming from checkpoint {lastest_ckpt_path}") + self.load_checkpoint(path=lastest_ckpt_path) + + # configure dataset + dataset: BaseImageDataset + dataset = hydra.utils.instantiate(cfg.task.dataset) + assert isinstance(dataset, BaseImageDataset) + train_dataloader = create_dataloader(dataset, **cfg.dataloader) + normalizer = dataset.get_normalizer() + + # configure validation dataset + val_dataset = dataset.get_validation_dataset() + val_dataloader = create_dataloader(val_dataset, **cfg.val_dataloader) + + self.model.set_normalizer(normalizer) + if cfg.training.use_ema: + self.ema_model.set_normalizer(normalizer) + + # configure lr scheduler + lr_scheduler = get_scheduler( + cfg.training.lr_scheduler, + optimizer=self.optimizer, + num_warmup_steps=cfg.training.lr_warmup_steps, + num_training_steps=(len(train_dataloader) * cfg.training.num_epochs) // + cfg.training.gradient_accumulate_every, + # pytorch assumes stepping LRScheduler every epoch + # however huggingface diffusers steps it every batch + last_epoch=self.global_step - 1, + ) + + # configure ema + ema: EMAModel = None + if cfg.training.use_ema: + ema = hydra.utils.instantiate(cfg.ema, model=self.ema_model) + + # configure env + # env_runner: BaseImageRunner + # env_runner = hydra.utils.instantiate( + # cfg.task.env_runner, + # output_dir=self.output_dir) + # assert isinstance(env_runner, BaseImageRunner) + env_runner = None + + # configure logging + # wandb_run = wandb.init( + # dir=str(self.output_dir), + # config=OmegaConf.to_container(cfg, resolve=True), + # **cfg.logging + # ) + # wandb.config.update( + # { + # "output_dir": self.output_dir, + # } + # ) + + # configure checkpoint + topk_manager = TopKCheckpointManager(save_dir=os.path.join(self.output_dir, "checkpoints"), + **cfg.checkpoint.topk) + + # device transfer + device = torch.device(cfg.training.device) + self.model.to(device) + if self.ema_model is not None: + self.ema_model.to(device) + optimizer_to(self.optimizer, device) + + # save batch for sampling + train_sampling_batch = None + + if cfg.training.debug: + cfg.training.num_epochs = 2 + cfg.training.max_train_steps = 3 + cfg.training.max_val_steps = 3 + cfg.training.rollout_every = 1 + cfg.training.checkpoint_every = 1 + cfg.training.val_every = 1 + cfg.training.sample_every = 1 + + # training loop + log_path = os.path.join(self.output_dir, "logs.json.txt") + + with JsonLogger(log_path) as json_logger: + for local_epoch_idx in range(cfg.training.num_epochs): + step_log = dict() + # ========= train for this epoch ========== + if cfg.training.freeze_encoder: + self.model.obs_encoder.eval() + self.model.obs_encoder.requires_grad_(False) + + train_losses = list() + with tqdm.tqdm( + train_dataloader, + desc=f"Training epoch {self.epoch}", + leave=False, + mininterval=cfg.training.tqdm_interval_sec, + ) as tepoch: + for batch_idx, batch in enumerate(tepoch): + batch = dataset.postprocess(batch, device) + if train_sampling_batch is None: + train_sampling_batch = batch + # compute loss + raw_loss = self.model.compute_loss(batch) + loss = raw_loss / cfg.training.gradient_accumulate_every + loss.backward() + + # step optimizer + if (self.global_step % cfg.training.gradient_accumulate_every == 0): + self.optimizer.step() + self.optimizer.zero_grad() + lr_scheduler.step() + + # update ema + if cfg.training.use_ema: + ema.step(self.model) + + # logging + raw_loss_cpu = raw_loss.item() + tepoch.set_postfix(loss=raw_loss_cpu, refresh=False) + train_losses.append(raw_loss_cpu) + step_log = { + "train_loss": raw_loss_cpu, + "global_step": self.global_step, + "epoch": self.epoch, + "lr": lr_scheduler.get_last_lr()[0], + } + + is_last_batch = batch_idx == (len(train_dataloader) - 1) + if not is_last_batch: + # log of last step is combined with validation and rollout + json_logger.log(step_log) + self.global_step += 1 + + if (cfg.training.max_train_steps + is not None) and batch_idx >= (cfg.training.max_train_steps - 1): + break + + # at the end of each epoch + # replace train_loss with epoch average + train_loss = np.mean(train_losses) + step_log["train_loss"] = train_loss + + # ========= eval for this epoch ========== + policy = self.model + if cfg.training.use_ema: + policy = self.ema_model + policy.eval() + + # run rollout + # if (self.epoch % cfg.training.rollout_every) == 0: + # runner_log = env_runner.run(policy) + # # log all + # step_log.update(runner_log) + + # run validation + if (self.epoch % cfg.training.val_every) == 0: + with torch.no_grad(): + val_losses = list() + with tqdm.tqdm( + val_dataloader, + desc=f"Validation epoch {self.epoch}", + leave=False, + mininterval=cfg.training.tqdm_interval_sec, + ) as tepoch: + for batch_idx, batch in enumerate(tepoch): + batch = dataset.postprocess(batch, device) + loss = self.model.compute_loss(batch) + val_losses.append(loss) + if (cfg.training.max_val_steps + is not None) and batch_idx >= (cfg.training.max_val_steps - 1): + break + if len(val_losses) > 0: + val_loss = torch.mean(torch.tensor(val_losses)).item() + # log epoch average validation loss + step_log["val_loss"] = val_loss + + # run diffusion sampling on a training batch + if (self.epoch % cfg.training.sample_every) == 0: + with torch.no_grad(): + # sample trajectory from training set, and evaluate difference + batch = train_sampling_batch + obs_dict = batch["obs"] + gt_action = batch["action"] + + result = policy.predict_action(obs_dict) + pred_action = result["action_pred"] + mse = torch.nn.functional.mse_loss(pred_action, gt_action) + step_log["train_action_mse_error"] = mse.item() + del batch + del obs_dict + del gt_action + del result + del pred_action + del mse + + # checkpoint + if ((self.epoch + 1) % cfg.training.checkpoint_every) == 0: + # checkpointing + save_name = pathlib.Path(self.cfg.task.dataset.zarr_path).stem + self.save_checkpoint(f"checkpoints/{save_name}-{seed}/{self.epoch + 1}.ckpt") # TODO + + # ========= eval end for this epoch ========== + policy.train() + + # end of epoch + # log of last step is combined with validation and rollout + json_logger.log(step_log) + self.global_step += 1 + self.epoch += 1 + + +class BatchSampler: + + def __init__( + self, + data_size: int, + batch_size: int, + shuffle: bool = False, + seed: int = 0, + drop_last: bool = True, + ): + assert drop_last + self.data_size = data_size + self.batch_size = batch_size + self.num_batch = data_size // batch_size + self.discard = data_size - batch_size * self.num_batch + self.shuffle = shuffle + self.rng = np.random.default_rng(seed) if shuffle else None + + def __iter__(self): + if self.shuffle: + perm = self.rng.permutation(self.data_size) + else: + perm = np.arange(self.data_size) + if self.discard > 0: + perm = perm[:-self.discard] + perm = perm.reshape(self.num_batch, self.batch_size) + for i in range(self.num_batch): + yield perm[i] + + def __len__(self): + return self.num_batch + + +def create_dataloader( + dataset, + *, + batch_size: int, + shuffle: bool, + num_workers: int, + pin_memory: bool, + persistent_workers: bool, + seed: int = 0, +): + batch_sampler = BatchSampler(len(dataset), batch_size, shuffle=shuffle, seed=seed, drop_last=True) + + def collate(x): + assert len(x) == 1 + return x[0] + + dataloader = DataLoader( + dataset, + collate_fn=collate, + sampler=batch_sampler, + num_workers=num_workers, + pin_memory=False, + persistent_workers=persistent_workers, + ) + return dataloader + + +@hydra.main( + version_base=None, + config_path=str(pathlib.Path(__file__).parent.parent.joinpath("config")), + config_name=pathlib.Path(__file__).stem, +) +def main(cfg): + workspace = RobotWorkspace(cfg) + workspace.run() + + +if __name__ == "__main__": + main() diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/dp_model.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/dp_model.py new file mode 100644 index 00000000..7a84fd46 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/dp_model.py @@ -0,0 +1,51 @@ +import numpy as np +import torch +import hydra +import dill +import sys, os + +current_file_path = os.path.abspath(__file__) +parent_dir = os.path.dirname(current_file_path) +sys.path.append(parent_dir) + +from diffusion_policy.workspace.robotworkspace import RobotWorkspace +from diffusion_policy.env_runner.dp_runner import DPRunner + +class DP: + + def __init__(self, ckpt_file: str, n_obs_steps, n_action_steps): + self.policy = self.get_policy(ckpt_file, None, "cuda:0") + self.runner = DPRunner(n_obs_steps=n_obs_steps, n_action_steps=n_action_steps) + + def update_obs(self, observation): + self.runner.update_obs(observation) + + def reset_obs(self): + self.runner.reset_obs() + + def get_action(self, observation=None): + action = self.runner.get_action(self.policy, observation) + return action + + def get_last_obs(self): + return self.runner.obs[-1] + + def get_policy(self, checkpoint, output_dir, device): + # load checkpoint + payload = torch.load(open(checkpoint, "rb"), pickle_module=dill) + cfg = payload["cfg"] + cls = hydra.utils.get_class(cfg._target_) + workspace = cls(cfg, output_dir=output_dir) + workspace: RobotWorkspace + workspace.load_payload(payload, exclude_keys=None, include_keys=None) + + # get policy from workspace + policy = workspace.model + if cfg.training.use_ema: + policy = workspace.ema_model + + device = torch.device(device) + policy.to(device) + policy.eval() + + return policy diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/eval.sh b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/eval.sh new file mode 100644 index 00000000..75365f9c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/eval.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# == keep unchanged == +policy_name=DP +task_name=${1} +task_config=${2} +ckpt_setting=${3} +expert_data_num=${4} +seed=${5} +gpu_id=${6} +DEBUG=False + +export CUDA_VISIBLE_DEVICES=${gpu_id} +echo -e "\033[33mgpu id (to use): ${gpu_id}\033[0m" + +cd ../.. + +PYTHONWARNINGS=ignore::UserWarning \ +python script/eval_policy.py --config policy/$policy_name/deploy_policy.yml \ + --overrides \ + --task_name ${task_name} \ + --task_config ${task_config} \ + --ckpt_setting ${ckpt_setting} \ + --expert_data_num ${expert_data_num} \ + --seed ${seed} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/eval_double_env.sh b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/eval_double_env.sh new file mode 100644 index 00000000..7c4e6305 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/eval_double_env.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# == keep unchanged == +policy_name=DP +task_name=${1} +task_config=${2} +ckpt_setting=${3} +expert_data_num=${4} +seed=${5} +gpu_id=${6} +policy_conda_env=${7} + +export CUDA_VISIBLE_DEVICES=${gpu_id} +echo -e "\033[33mgpu id (to use): ${gpu_id}\033[0m" + +cd ../.. + +yaml_file="policy/${policy_name}/deploy_policy.yml" + +echo "policy_conda_env is '$policy_conda_env'" + +# Find an available port +FREE_PORT=$(python3 - << 'EOF' +import socket +with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(('', 0)) + print(s.getsockname()[1]) +EOF +) +echo -e "\033[33mUsing socket port: ${FREE_PORT}\033[0m" + +# Start the server in the background +echo -e "\033[32m[server] Activating Conda environment: ${policy_conda_env}\033[0m" +source "$(conda info --base)/etc/profile.d/conda.sh" +conda activate "${policy_conda_env}" + +echo -e "\033[32m[server] Launching policy_model_server (PID will be recorded)...\033[0m" +PYTHONWARNINGS=ignore::UserWarning \ +python script/policy_model_server.py \ + --port ${FREE_PORT} \ + --policy_conda_env ${policy_conda_env} \ + --config policy/${policy_name}/deploy_policy.yml \ + --overrides \ + --task_name ${task_name} \ + --task_config ${task_config} \ + --ckpt_setting ${ckpt_setting} \ + --expert_data_num ${expert_data_num} \ + --seed ${seed} & +SERVER_PID=$! + +# Ensure the server is killed when this script exits +trap "echo -e '\033[31m[cleanup] Killing server (PID=${SERVER_PID})\033[0m'; kill ${SERVER_PID} 2>/dev/null" EXIT + +conda deactivate + +# Start the client in the foreground +echo -e "\033[34m[client] Starting eval_policy_client on port ${FREE_PORT}...\033[0m" +PYTHONWARNINGS=ignore::UserWarning \ +python script/eval_policy_client.py \ + --port ${FREE_PORT} \ + --policy_conda_env ${policy_conda_env} \ + --config policy/${policy_name}/deploy_policy.yml \ + --overrides \ + --task_name ${task_name} \ + --task_config ${task_config} \ + --ckpt_setting ${ckpt_setting} \ + --expert_data_num ${expert_data_num} \ + --seed ${seed} + +echo -e "\033[33m[main] eval_policy_client has finished; the server will be terminated.\033[0m" diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/process_data.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/process_data.py new file mode 100644 index 00000000..347ac8fb --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/process_data.py @@ -0,0 +1,158 @@ +import pickle, os +import numpy as np +import pdb +from copy import deepcopy +import zarr +import shutil +import argparse +import yaml +import cv2 +import h5py + + +def load_hdf5(dataset_path): + if not os.path.isfile(dataset_path): + print(f"Dataset does not exist at \n{dataset_path}\n") + exit() + + with h5py.File(dataset_path, "r") as root: + left_gripper, left_arm = ( + root["/joint_action/left_gripper"][()], + root["/joint_action/left_arm"][()], + ) + right_gripper, right_arm = ( + root["/joint_action/right_gripper"][()], + root["/joint_action/right_arm"][()], + ) + vector = root["/joint_action/vector"][()] + image_dict = dict() + for cam_name in root[f"/observation/"].keys(): + image_dict[cam_name] = root[f"/observation/{cam_name}/rgb"][()] + + return left_gripper, left_arm, right_gripper, right_arm, vector, image_dict + + +def main(): + parser = argparse.ArgumentParser(description="Process some episodes.") + parser.add_argument( + "task_name", + type=str, + help="The name of the task (e.g., beat_block_hammer)", + ) + parser.add_argument("task_config", type=str) + parser.add_argument( + "expert_data_num", + type=int, + help="Number of episodes to process (e.g., 50)", + ) + args = parser.parse_args() + + task_name = args.task_name + num = args.expert_data_num + task_config = args.task_config + + load_dir = "../../data/" + str(task_name) + "/" + str(task_config) + + total_count = 0 + + save_dir = f"./data/{task_name}-{task_config}-{num}.zarr" + + if os.path.exists(save_dir): + shutil.rmtree(save_dir) + + current_ep = 0 + + zarr_root = zarr.group(save_dir) + zarr_data = zarr_root.create_group("data") + zarr_meta = zarr_root.create_group("meta") + + head_camera_arrays, front_camera_arrays, left_camera_arrays, right_camera_arrays = ( + [], + [], + [], + [], + ) + episode_ends_arrays, action_arrays, state_arrays, joint_action_arrays = ( + [], + [], + [], + [], + ) + + while current_ep < num: + print(f"processing episode: {current_ep + 1} / {num}", end="\r") + + load_path = os.path.join(load_dir, f"data/episode{current_ep}.hdf5") + ( + left_gripper_all, + left_arm_all, + right_gripper_all, + right_arm_all, + vector_all, + image_dict_all, + ) = load_hdf5(load_path) + + for j in range(0, left_gripper_all.shape[0]): + + head_img_bit = image_dict_all["head_camera"][j] + joint_state = vector_all[j] + + if j != left_gripper_all.shape[0] - 1: + head_img = cv2.imdecode(np.frombuffer(head_img_bit, np.uint8), cv2.IMREAD_COLOR) + head_camera_arrays.append(head_img) + state_arrays.append(joint_state) + if j != 0: + joint_action_arrays.append(joint_state) + + current_ep += 1 + total_count += left_gripper_all.shape[0] - 1 + episode_ends_arrays.append(total_count) + + print() + episode_ends_arrays = np.array(episode_ends_arrays) + # action_arrays = np.array(action_arrays) + state_arrays = np.array(state_arrays) + head_camera_arrays = np.array(head_camera_arrays) + joint_action_arrays = np.array(joint_action_arrays) + + head_camera_arrays = np.moveaxis(head_camera_arrays, -1, 1) # NHWC -> NCHW + + compressor = zarr.Blosc(cname="zstd", clevel=3, shuffle=1) + # action_chunk_size = (100, action_arrays.shape[1]) + state_chunk_size = (100, state_arrays.shape[1]) + joint_chunk_size = (100, joint_action_arrays.shape[1]) + head_camera_chunk_size = (100, *head_camera_arrays.shape[1:]) + zarr_data.create_dataset( + "head_camera", + data=head_camera_arrays, + chunks=head_camera_chunk_size, + overwrite=True, + compressor=compressor, + ) + zarr_data.create_dataset( + "state", + data=state_arrays, + chunks=state_chunk_size, + dtype="float32", + overwrite=True, + compressor=compressor, + ) + zarr_data.create_dataset( + "action", + data=joint_action_arrays, + chunks=joint_chunk_size, + dtype="float32", + overwrite=True, + compressor=compressor, + ) + zarr_meta.create_dataset( + "episode_ends", + data=episode_ends_arrays, + dtype="int64", + overwrite=True, + compressor=compressor, + ) + + +if __name__ == "__main__": + main() diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/process_data.sh b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/process_data.sh new file mode 100644 index 00000000..67a93d3a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/process_data.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +task_name=${1} +task_config=${2} +expert_data_num=${3} + +python process_data.py $task_name $task_config $expert_data_num \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/pyproject.toml b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/pyproject.toml new file mode 100644 index 00000000..ba2028ff --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/pyproject.toml @@ -0,0 +1,13 @@ +[build-system] +requires = ["flit_core >=3.7,<4"] +build-backend = "flit_core.buildapi" + +[project] +name = "diffusion_policy" +version = "0.1.0" +description = "Diffusion policy for RoboTwin" +requires-python = ">=3.8" +dependencies = [ + "hydra-core==1.2.0", + "numba" +] \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/train.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/train.py new file mode 100644 index 00000000..d9e2110c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/train.py @@ -0,0 +1,70 @@ +""" +Usage: +Training: +python train.py --config-name=train_diffusion_lowdim_workspace +""" + +import sys + +# use line-buffering for both stdout and stderr +sys.stdout = open(sys.stdout.fileno(), mode="w", buffering=1) +sys.stderr = open(sys.stderr.fileno(), mode="w", buffering=1) + +import hydra, pdb +from omegaconf import OmegaConf +import pathlib, yaml +from diffusion_policy.workspace.base_workspace import BaseWorkspace + +import os + +current_file_path = os.path.abspath(__file__) +parent_directory = os.path.dirname(current_file_path) + + +def get_camera_config(camera_type): + camera_config_path = os.path.join(parent_directory, "../../task_config/_camera_config.yml") + + assert os.path.isfile(camera_config_path), "task config file is missing" + + with open(camera_config_path, "r", encoding="utf-8") as f: + args = yaml.load(f.read(), Loader=yaml.FullLoader) + + assert camera_type in args, f"camera {camera_type} is not defined" + return args[camera_type] + + +# allows arbitrary python code execution in configs using the ${eval:''} resolver +OmegaConf.register_new_resolver("eval", eval, replace=True) + + +@hydra.main( + version_base=None, + config_path=str(pathlib.Path(__file__).parent.joinpath("diffusion_policy", "config")), +) +def main(cfg: OmegaConf): + # resolve immediately so all the ${now:} resolvers + # will use the same time. + head_camera_type = cfg.head_camera_type + head_camera_cfg = get_camera_config(head_camera_type) + cfg.task.image_shape = [3, head_camera_cfg["h"], head_camera_cfg["w"]] + cfg.task.shape_meta.obs.head_cam.shape = [ + 3, + head_camera_cfg["h"], + head_camera_cfg["w"], + ] + OmegaConf.resolve(cfg) + cfg.task.image_shape = [3, head_camera_cfg["h"], head_camera_cfg["w"]] + cfg.task.shape_meta.obs.head_cam.shape = [ + 3, + head_camera_cfg["h"], + head_camera_cfg["w"], + ] + + cls = hydra.utils.get_class(cfg._target_) + workspace: BaseWorkspace = cls(cfg) + print(cfg.task.dataset.zarr_path, cfg.task_name) + workspace.run() + + +if __name__ == "__main__": + main() diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/train.sh b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/train.sh new file mode 100644 index 00000000..783b02d9 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP/train.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +task_name=${1} +task_config=${2} +expert_data_num=${3} +seed=${4} +action_dim=${5} +gpu_id=${6} + +head_camera_type=D435 + +DEBUG=False +save_ckpt=True + +alg_name=robot_dp_$action_dim +config_name=${alg_name} +addition_info=train +exp_name=${task_name}-robot_dp-${addition_info} +run_dir="data/outputs/${exp_name}_seed${seed}" + +echo -e "\033[33mgpu id (to use): ${gpu_id}\033[0m" + + +if [ $DEBUG = True ]; then + wandb_mode=offline + # wandb_mode=online + echo -e "\033[33mDebug mode!\033[0m" + echo -e "\033[33mDebug mode!\033[0m" + echo -e "\033[33mDebug mode!\033[0m" +else + wandb_mode=online + echo -e "\033[33mTrain mode\033[0m" +fi + +export HYDRA_FULL_ERROR=1 +export CUDA_VISIBLE_DEVICES=${gpu_id} + +if [ ! -d "./data/${task_name}-${task_config}-${expert_data_num}.zarr" ]; then + bash process_data.sh ${task_name} ${task_config} ${expert_data_num} +fi + +python train.py --config-name=${config_name}.yaml \ + task.name=${task_name} \ + task.dataset.zarr_path="data/${task_name}-${task_config}-${expert_data_num}.zarr" \ + training.debug=$DEBUG \ + training.seed=${seed} \ + training.device="cuda:0" \ + exp_name=${exp_name} \ + logging.mode=${wandb_mode} \ + setting=${task_config} \ + expert_data_num=${expert_data_num} \ + head_camera_type=$head_camera_type + # checkpoint.save_ckpt=${save_ckpt} + # hydra.run.dir=${run_dir} \ \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/.gitignore b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/.gitignore new file mode 100644 index 00000000..296383a8 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/.gitignore @@ -0,0 +1,5 @@ +3D-Diffusion-Policy/data/* +third_party/ +third_party/pytorch3d +checkpoints/* +data/* \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/.gitignore b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/.gitignore new file mode 100644 index 00000000..dfb0916b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/.gitignore @@ -0,0 +1,142 @@ +bin +logs +wandb +outputs +data +data_local +.vscode +_wandb + +**/.DS_Store + +fuse.cfg + +*.ai + +# Generation results +results/ + +ray/auth.json + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +/data/RoboTwin_private/policy/3D-Diffusion-Policy/3D-Diffusion-Policy/diffusion_policy_3d/config/robot_dp3.yaml \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/__init__.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/common/checkpoint_util.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/common/checkpoint_util.py new file mode 100644 index 00000000..f3ce00a0 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/common/checkpoint_util.py @@ -0,0 +1,61 @@ +from typing import Optional, Dict +import os + + +class TopKCheckpointManager: + + def __init__( + self, + save_dir, + monitor_key: str, + mode="min", + k=1, + format_str="epoch={epoch:03d}-train_loss={train_loss:.3f}.ckpt", + ): + assert mode in ["max", "min"] + assert k >= 0 + + self.save_dir = save_dir + self.monitor_key = monitor_key + self.mode = mode + self.k = k + self.format_str = format_str + self.path_value_map = dict() + + def get_ckpt_path(self, data: Dict[str, float]) -> Optional[str]: + if self.k == 0: + return None + + value = data[self.monitor_key] + ckpt_path = os.path.join(self.save_dir, self.format_str.format(**data)) + + if len(self.path_value_map) < self.k: + # under-capacity + self.path_value_map[ckpt_path] = value + return ckpt_path + + # at capacity + sorted_map = sorted(self.path_value_map.items(), key=lambda x: x[1]) + min_path, min_value = sorted_map[0] + max_path, max_value = sorted_map[-1] + + delete_path = None + if self.mode == "max": + if value > min_value: + delete_path = min_path + else: + if value < max_value: + delete_path = max_path + + if delete_path is None: + return None + else: + del self.path_value_map[delete_path] + self.path_value_map[ckpt_path] = value + + if not os.path.exists(self.save_dir): + os.mkdir(self.save_dir) + + if os.path.exists(delete_path): + os.remove(delete_path) + return ckpt_path diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/common/logger_util.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/common/logger_util.py new file mode 100644 index 00000000..faf59165 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/common/logger_util.py @@ -0,0 +1,51 @@ +import heapq + + +class LargestKRecorder: + + def __init__(self, K): + """ + Initialize the EfficientScalarRecorder. + + Parameters: + - K: Number of largest scalars to consider when computing the average. + """ + self.scalars = [] + self.K = K + + def record(self, scalar): + """ + Record a scalar value. + + Parameters: + - scalar: The scalar value to be recorded. + """ + if len(self.scalars) < self.K: + heapq.heappush(self.scalars, scalar) + else: + # Compare the new scalar with the smallest value in the heap + if scalar > self.scalars[0]: + heapq.heappushpop(self.scalars, scalar) + + def average_of_largest_K(self): + """ + Compute the average of the largest K scalar values recorded. + + Returns: + - avg: Average of the largest K scalars. + """ + if len(self.scalars) == 0: + raise ValueError("No scalars have been recorded yet.") + + return sum(self.scalars) / len(self.scalars) + + +# Example Usage: +# recorder = EfficientScalarRecorder(K=5) +# recorder.record(1) +# recorder.record(2) +# recorder.record(3) +# recorder.record(4) +# recorder.record(5) +# recorder.record(6) +# print(recorder.average_of_largest_K()) # Expected output: (6 + 5 + 4 + 3 + 2) / 5 = 4.0 diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/common/model_util.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/common/model_util.py new file mode 100644 index 00000000..85fb0741 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/common/model_util.py @@ -0,0 +1,26 @@ +from termcolor import cprint + + +def print_params(model): + """ + Print the number of parameters in each part of the model. + """ + params_dict = {} + + all_num_param = sum(p.numel() for p in model.parameters()) + + for name, param in model.named_parameters(): + part_name = name.split(".")[0] + if part_name not in params_dict: + params_dict[part_name] = 0 + params_dict[part_name] += param.numel() + + cprint(f"----------------------------------", "cyan") + cprint(f"Class name: {model.__class__.__name__}", "cyan") + cprint(f" Number of parameters: {all_num_param / 1e6:.4f}M", "cyan") + for part_name, num_params in params_dict.items(): + cprint( + f" {part_name}: {num_params / 1e6:.4f}M ({num_params / all_num_param:.2%})", + "cyan", + ) + cprint(f"----------------------------------", "cyan") diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/common/pytorch_util.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/common/pytorch_util.py new file mode 100644 index 00000000..ef9e8250 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/common/pytorch_util.py @@ -0,0 +1,49 @@ +from typing import Dict, Callable, List +import collections +import torch +import torch.nn as nn + + +def dict_apply(x: Dict[str, torch.Tensor], func: Callable[[torch.Tensor], torch.Tensor]) -> Dict[str, torch.Tensor]: + result = dict() + for key, value in x.items(): + if isinstance(value, dict): + result[key] = dict_apply(value, func) + else: + result[key] = func(value) + return result + + +def pad_remaining_dims(x, target): + assert x.shape == target.shape[:len(x.shape)] + return x.reshape(x.shape + (1, ) * (len(target.shape) - len(x.shape))) + + +def dict_apply_split( + x: Dict[str, torch.Tensor], + split_func: Callable[[torch.Tensor], Dict[str, torch.Tensor]], +) -> Dict[str, torch.Tensor]: + results = collections.defaultdict(dict) + for key, value in x.items(): + result = split_func(value) + for k, v in result.items(): + results[k][key] = v + return results + + +def dict_apply_reduce( + x: List[Dict[str, torch.Tensor]], + reduce_func: Callable[[List[torch.Tensor]], torch.Tensor], +) -> Dict[str, torch.Tensor]: + result = dict() + for key in x[0].keys(): + result[key] = reduce_func([x_[key] for x_ in x]) + return result + + +def optimizer_to(optimizer, device): + for state in optimizer.state.values(): + for k, v in state.items(): + if isinstance(v, torch.Tensor): + state[k] = v.to(device=device) + return optimizer diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/common/replay_buffer.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/common/replay_buffer.py new file mode 100644 index 00000000..bd4aa1e6 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/common/replay_buffer.py @@ -0,0 +1,628 @@ +from typing import Union, Dict, Optional +import os +import math +import numbers +import zarr +import numcodecs +import numpy as np +from functools import cached_property +from termcolor import cprint + + +def check_chunks_compatible(chunks: tuple, shape: tuple): + assert len(shape) == len(chunks) + for c in chunks: + assert isinstance(c, numbers.Integral) + assert c > 0 + + +def rechunk_recompress_array(group, name, chunks=None, chunk_length=None, compressor=None, tmp_key="_temp"): + old_arr = group[name] + if chunks is None: + if chunk_length is not None: + chunks = (chunk_length, ) + old_arr.chunks[1:] + else: + chunks = old_arr.chunks + check_chunks_compatible(chunks, old_arr.shape) + + if compressor is None: + compressor = old_arr.compressor + + if (chunks == old_arr.chunks) and (compressor == old_arr.compressor): + # no change + return old_arr + + # rechunk recompress + group.move(name, tmp_key) + old_arr = group[tmp_key] + n_copied, n_skipped, n_bytes_copied = zarr.copy( + source=old_arr, + dest=group, + name=name, + chunks=chunks, + compressor=compressor, + ) + del group[tmp_key] + arr = group[name] + return arr + + +def get_optimal_chunks(shape, dtype, target_chunk_bytes=2e6, max_chunk_length=None): + """ + Common shapes + T,D + T,N,D + T,H,W,C + T,N,H,W,C + """ + itemsize = np.dtype(dtype).itemsize + # reversed + rshape = list(shape[::-1]) + if max_chunk_length is not None: + rshape[-1] = int(max_chunk_length) + split_idx = len(shape) - 1 + for i in range(len(shape) - 1): + this_chunk_bytes = itemsize * np.prod(rshape[:i]) + next_chunk_bytes = itemsize * np.prod(rshape[:i + 1]) + if (this_chunk_bytes <= target_chunk_bytes and next_chunk_bytes > target_chunk_bytes): + split_idx = i + + rchunks = rshape[:split_idx] + item_chunk_bytes = itemsize * np.prod(rshape[:split_idx]) + this_max_chunk_length = rshape[split_idx] + next_chunk_length = min(this_max_chunk_length, math.ceil(target_chunk_bytes / item_chunk_bytes)) + rchunks.append(next_chunk_length) + len_diff = len(shape) - len(rchunks) + rchunks.extend([1] * len_diff) + chunks = tuple(rchunks[::-1]) + # print(np.prod(chunks) * itemsize / target_chunk_bytes) + return chunks + + +class ReplayBuffer: + """ + Zarr-based temporal datastructure. + Assumes first dimension to be time. Only chunk in time dimension. + """ + + def __init__(self, root: Union[zarr.Group, Dict[str, dict]]): + """ + Dummy constructor. Use copy_from* and create_from* class methods instead. + """ + assert "data" in root + assert "meta" in root + assert "episode_ends" in root["meta"] + for key, value in root["data"].items(): + assert value.shape[0] == root["meta"]["episode_ends"][-1] + self.root = root + + # ============= create constructors =============== + @classmethod + def create_empty_zarr(cls, storage=None, root=None): + if root is None: + if storage is None: + storage = zarr.MemoryStore() + root = zarr.group(store=storage) + data = root.require_group("data", overwrite=False) + meta = root.require_group("meta", overwrite=False) + if "episode_ends" not in meta: + episode_ends = meta.zeros( + "episode_ends", + shape=(0, ), + dtype=np.int64, + compressor=None, + overwrite=False, + ) + return cls(root=root) + + @classmethod + def create_empty_numpy(cls): + root = { + "data": dict(), + "meta": { + "episode_ends": np.zeros((0, ), dtype=np.int64) + }, + } + return cls(root=root) + + @classmethod + def create_from_group(cls, group, **kwargs): + if "data" not in group: + # create from stratch + buffer = cls.create_empty_zarr(root=group, **kwargs) + else: + # already exist + buffer = cls(root=group, **kwargs) + return buffer + + @classmethod + def create_from_path(cls, zarr_path, mode="r", **kwargs): + """ + Open a on-disk zarr directly (for dataset larger than memory). + Slower. + """ + group = zarr.open(os.path.expanduser(zarr_path), mode) + return cls.create_from_group(group, **kwargs) + + # ============= copy constructors =============== + @classmethod + def copy_from_store( + cls, + src_store, + store=None, + keys=None, + chunks: Dict[str, tuple] = dict(), + compressors: Union[dict, str, numcodecs.abc.Codec] = dict(), + if_exists="replace", + **kwargs, + ): + """ + Load to memory. + """ + src_root = zarr.group(src_store) + root = None + if store is None: + # numpy backend + meta = dict() + for key, value in src_root["meta"].items(): + if len(value.shape) == 0: + meta[key] = np.array(value) + else: + meta[key] = value[:] + + if keys is None: + keys = src_root["data"].keys() + data = dict() + for key in keys: + arr = src_root["data"][key] + data[key] = arr[:] + root = {"meta": meta, "data": data} + else: + root = zarr.group(store=store) + # copy without recompression + n_copied, n_skipped, n_bytes_copied = zarr.copy_store( + source=src_store, + dest=store, + source_path="/meta", + dest_path="/meta", + if_exists=if_exists, + ) + data_group = root.create_group("data", overwrite=True) + if keys is None: + keys = src_root["data"].keys() + for key in keys: + value = src_root["data"][key] + cks = cls._resolve_array_chunks(chunks=chunks, key=key, array=value) + cpr = cls._resolve_array_compressor(compressors=compressors, key=key, array=value) + if cks == value.chunks and cpr == value.compressor: + # copy without recompression + this_path = "/data/" + key + n_copied, n_skipped, n_bytes_copied = zarr.copy_store( + source=src_store, + dest=store, + source_path=this_path, + dest_path=this_path, + if_exists=if_exists, + ) + else: + # copy with recompression + n_copied, n_skipped, n_bytes_copied = zarr.copy( + source=value, + dest=data_group, + name=key, + chunks=cks, + compressor=cpr, + if_exists=if_exists, + ) + buffer = cls(root=root) + for key, value in buffer.items(): + cprint( + f"Replay Buffer: {key}, shape {value.shape}, dtype {value.dtype}, range {value.min():.2f}~{value.max():.2f}", + "green", + ) + cprint("--------------------------", "green") + return buffer + + @classmethod + def copy_from_path( + cls, + zarr_path, + backend=None, + store=None, + keys=None, + chunks: Dict[str, tuple] = dict(), + compressors: Union[dict, str, numcodecs.abc.Codec] = dict(), + if_exists="replace", + **kwargs, + ): + """ + Copy a on-disk zarr to in-memory compressed. + Recommended + """ + if backend == "numpy": + print("backend argument is deprecated!") + store = None + group = zarr.open(os.path.expanduser(zarr_path), "r") + return cls.copy_from_store( + src_store=group.store, + store=store, + keys=keys, + chunks=chunks, + compressors=compressors, + if_exists=if_exists, + **kwargs, + ) + + # ============= save methods =============== + def save_to_store( + self, + store, + chunks: Optional[Dict[str, tuple]] = dict(), + compressors: Union[str, numcodecs.abc.Codec, dict] = dict(), + if_exists="replace", + **kwargs, + ): + + root = zarr.group(store) + if self.backend == "zarr": + # recompression free copy + n_copied, n_skipped, n_bytes_copied = zarr.copy_store( + source=self.root.store, + dest=store, + source_path="/meta", + dest_path="/meta", + if_exists=if_exists, + ) + else: + meta_group = root.create_group("meta", overwrite=True) + # save meta, no chunking + for key, value in self.root["meta"].items(): + _ = meta_group.array(name=key, data=value, shape=value.shape, chunks=value.shape) + + # save data, chunk + data_group = root.create_group("data", overwrite=True) + for key, value in self.root["data"].items(): + cks = self._resolve_array_chunks(chunks=chunks, key=key, array=value) + cpr = self._resolve_array_compressor(compressors=compressors, key=key, array=value) + if isinstance(value, zarr.Array): + if cks == value.chunks and cpr == value.compressor: + # copy without recompression + this_path = "/data/" + key + n_copied, n_skipped, n_bytes_copied = zarr.copy_store( + source=self.root.store, + dest=store, + source_path=this_path, + dest_path=this_path, + if_exists=if_exists, + ) + else: + # copy with recompression + n_copied, n_skipped, n_bytes_copied = zarr.copy( + source=value, + dest=data_group, + name=key, + chunks=cks, + compressor=cpr, + if_exists=if_exists, + ) + else: + # numpy + _ = data_group.array(name=key, data=value, chunks=cks, compressor=cpr) + return store + + def save_to_path( + self, + zarr_path, + chunks: Optional[Dict[str, tuple]] = dict(), + compressors: Union[str, numcodecs.abc.Codec, dict] = dict(), + if_exists="replace", + **kwargs, + ): + store = zarr.DirectoryStore(os.path.expanduser(zarr_path)) + return self.save_to_store(store, chunks=chunks, compressors=compressors, if_exists=if_exists, **kwargs) + + @staticmethod + def resolve_compressor(compressor="default"): + if compressor == "default": + compressor = numcodecs.Blosc(cname="lz4", clevel=5, shuffle=numcodecs.Blosc.NOSHUFFLE) + elif compressor == "disk": + compressor = numcodecs.Blosc("zstd", clevel=5, shuffle=numcodecs.Blosc.BITSHUFFLE) + return compressor + + @classmethod + def _resolve_array_compressor(cls, compressors: Union[dict, str, numcodecs.abc.Codec], key, array): + # allows compressor to be explicitly set to None + cpr = "nil" + if isinstance(compressors, dict): + if key in compressors: + cpr = cls.resolve_compressor(compressors[key]) + elif isinstance(array, zarr.Array): + cpr = array.compressor + else: + cpr = cls.resolve_compressor(compressors) + # backup default + if cpr == "nil": + cpr = cls.resolve_compressor("default") + return cpr + + @classmethod + def _resolve_array_chunks(cls, chunks: Union[dict, tuple], key, array): + cks = None + if isinstance(chunks, dict): + if key in chunks: + cks = chunks[key] + elif isinstance(array, zarr.Array): + cks = array.chunks + elif isinstance(chunks, tuple): + cks = chunks + else: + raise TypeError(f"Unsupported chunks type {type(chunks)}") + # backup default + if cks is None: + cks = get_optimal_chunks(shape=array.shape, dtype=array.dtype) + # check + check_chunks_compatible(chunks=cks, shape=array.shape) + return cks + + # ============= properties ================= + @cached_property + def data(self): + return self.root["data"] + + @cached_property + def meta(self): + return self.root["meta"] + + def update_meta(self, data): + # sanitize data + np_data = dict() + for key, value in data.items(): + if isinstance(value, np.ndarray): + np_data[key] = value + else: + arr = np.array(value) + if arr.dtype == object: + raise TypeError(f"Invalid value type {type(value)}") + np_data[key] = arr + + meta_group = self.meta + if self.backend == "zarr": + for key, value in np_data.items(): + _ = meta_group.array( + name=key, + data=value, + shape=value.shape, + chunks=value.shape, + overwrite=True, + ) + else: + meta_group.update(np_data) + + return meta_group + + @property + def episode_ends(self): + return self.meta["episode_ends"] + + def get_episode_idxs(self): + import numba + + numba.jit(nopython=True) + + def _get_episode_idxs(episode_ends): + result = np.zeros((episode_ends[-1], ), dtype=np.int64) + for i in range(len(episode_ends)): + start = 0 + if i > 0: + start = episode_ends[i - 1] + end = episode_ends[i] + for idx in range(start, end): + result[idx] = i + return result + + return _get_episode_idxs(self.episode_ends) + + @property + def backend(self): + backend = "numpy" + if isinstance(self.root, zarr.Group): + backend = "zarr" + return backend + + # =========== dict-like API ============== + def __repr__(self) -> str: + if self.backend == "zarr": + return str(self.root.tree()) + else: + return super().__repr__() + + def keys(self): + return self.data.keys() + + def values(self): + return self.data.values() + + def items(self): + return self.data.items() + + def __getitem__(self, key): + return self.data[key] + + def __contains__(self, key): + return key in self.data + + # =========== our API ============== + @property + def n_steps(self): + if len(self.episode_ends) == 0: + return 0 + return self.episode_ends[-1] + + @property + def n_episodes(self): + return len(self.episode_ends) + + @property + def chunk_size(self): + if self.backend == "zarr": + return next(iter(self.data.arrays()))[-1].chunks[0] + return None + + @property + def episode_lengths(self): + ends = self.episode_ends[:] + ends = np.insert(ends, 0, 0) + lengths = np.diff(ends) + return lengths + + def add_episode( + self, + data: Dict[str, np.ndarray], + chunks: Optional[Dict[str, tuple]] = dict(), + compressors: Union[str, numcodecs.abc.Codec, dict] = dict(), + ): + assert len(data) > 0 + is_zarr = self.backend == "zarr" + + curr_len = self.n_steps + episode_length = None + for key, value in data.items(): + assert len(value.shape) >= 1 + if episode_length is None: + episode_length = len(value) + else: + assert episode_length == len(value) + new_len = curr_len + episode_length + + for key, value in data.items(): + new_shape = (new_len, ) + value.shape[1:] + # create array + if key not in self.data: + if is_zarr: + cks = self._resolve_array_chunks(chunks=chunks, key=key, array=value) + cpr = self._resolve_array_compressor(compressors=compressors, key=key, array=value) + arr = self.data.zeros( + name=key, + shape=new_shape, + chunks=cks, + dtype=value.dtype, + compressor=cpr, + ) + else: + # copy data to prevent modify + arr = np.zeros(shape=new_shape, dtype=value.dtype) + self.data[key] = arr + else: + arr = self.data[key] + assert value.shape[1:] == arr.shape[1:] + # same method for both zarr and numpy + if is_zarr: + arr.resize(new_shape) + else: + arr.resize(new_shape, refcheck=False) + # copy data + arr[-value.shape[0]:] = value + + # append to episode ends + episode_ends = self.episode_ends + if is_zarr: + episode_ends.resize(episode_ends.shape[0] + 1) + else: + episode_ends.resize(episode_ends.shape[0] + 1, refcheck=False) + episode_ends[-1] = new_len + + # rechunk + if is_zarr: + if episode_ends.chunks[0] < episode_ends.shape[0]: + rechunk_recompress_array( + self.meta, + "episode_ends", + chunk_length=int(episode_ends.shape[0] * 1.5), + ) + + def drop_episode(self): + is_zarr = self.backend == "zarr" + episode_ends = self.episode_ends[:].copy() + assert len(episode_ends) > 0 + start_idx = 0 + if len(episode_ends) > 1: + start_idx = episode_ends[-2] + for key, value in self.data.items(): + new_shape = (start_idx, ) + value.shape[1:] + if is_zarr: + value.resize(new_shape) + else: + value.resize(new_shape, refcheck=False) + if is_zarr: + self.episode_ends.resize(len(episode_ends) - 1) + else: + self.episode_ends.resize(len(episode_ends) - 1, refcheck=False) + + def pop_episode(self): + assert self.n_episodes > 0 + episode = self.get_episode(self.n_episodes - 1, copy=True) + self.drop_episode() + return episode + + def extend(self, data): + self.add_episode(data) + + def get_episode(self, idx, copy=False): + idx = list(range(len(self.episode_ends)))[idx] + start_idx = 0 + if idx > 0: + start_idx = self.episode_ends[idx - 1] + end_idx = self.episode_ends[idx] + result = self.get_steps_slice(start_idx, end_idx, copy=copy) + return result + + def get_episode_slice(self, idx): + start_idx = 0 + if idx > 0: + start_idx = self.episode_ends[idx - 1] + end_idx = self.episode_ends[idx] + return slice(start_idx, end_idx) + + def get_steps_slice(self, start, stop, step=None, copy=False): + _slice = slice(start, stop, step) + + result = dict() + for key, value in self.data.items(): + x = value[_slice] + if copy and isinstance(value, np.ndarray): + x = x.copy() + result[key] = x + return result + + # =========== chunking ============= + def get_chunks(self) -> dict: + assert self.backend == "zarr" + chunks = dict() + for key, value in self.data.items(): + chunks[key] = value.chunks + return chunks + + def set_chunks(self, chunks: dict): + assert self.backend == "zarr" + for key, value in chunks.items(): + if key in self.data: + arr = self.data[key] + if value != arr.chunks: + check_chunks_compatible(chunks=value, shape=arr.shape) + rechunk_recompress_array(self.data, key, chunks=value) + + def get_compressors(self) -> dict: + assert self.backend == "zarr" + compressors = dict() + for key, value in self.data.items(): + compressors[key] = value.compressor + return compressors + + def set_compressors(self, compressors: dict): + assert self.backend == "zarr" + for key, value in compressors.items(): + if key in self.data: + arr = self.data[key] + compressor = self.resolve_compressor(value) + if compressor != arr.compressor: + rechunk_recompress_array(self.data, key, compressor=compressor) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/common/sampler.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/common/sampler.py new file mode 100644 index 00000000..2c57d656 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/common/sampler.py @@ -0,0 +1,163 @@ +from typing import Optional +import numpy as np +import numba +from diffusion_policy_3d.common.replay_buffer import ReplayBuffer + + +@numba.jit(nopython=True) +def create_indices( + episode_ends: np.ndarray, + sequence_length: int, + episode_mask: np.ndarray, + pad_before: int = 0, + pad_after: int = 0, + debug: bool = True, +) -> np.ndarray: + episode_mask.shape == episode_ends.shape + pad_before = min(max(pad_before, 0), sequence_length - 1) + pad_after = min(max(pad_after, 0), sequence_length - 1) + + indices = list() + for i in range(len(episode_ends)): + if not episode_mask[i]: + # skip episode + continue + start_idx = 0 + if i > 0: + start_idx = episode_ends[i - 1] + end_idx = episode_ends[i] + episode_length = end_idx - start_idx + + min_start = -pad_before + max_start = episode_length - sequence_length + pad_after + + # range stops one idx before end + for idx in range(min_start, max_start + 1): + buffer_start_idx = max(idx, 0) + start_idx + buffer_end_idx = min(idx + sequence_length, episode_length) + start_idx + start_offset = buffer_start_idx - (idx + start_idx) + end_offset = (idx + sequence_length + start_idx) - buffer_end_idx + sample_start_idx = 0 + start_offset + sample_end_idx = sequence_length - end_offset + if debug: + assert start_offset >= 0 + assert end_offset >= 0 + assert (sample_end_idx - sample_start_idx) == (buffer_end_idx - buffer_start_idx) + indices.append([buffer_start_idx, buffer_end_idx, sample_start_idx, sample_end_idx]) + indices = np.array(indices) + return indices + + +def get_val_mask(n_episodes, val_ratio, seed=0): + val_mask = np.zeros(n_episodes, dtype=bool) + if val_ratio <= 0: + return val_mask + + # have at least 1 episode for validation, and at least 1 episode for train + n_val = min(max(1, round(n_episodes * val_ratio)), n_episodes - 1) + rng = np.random.default_rng(seed=seed) + val_idxs = rng.choice(n_episodes, size=n_val, replace=False) + val_mask[val_idxs] = True + return val_mask + + +def downsample_mask(mask, max_n, seed=0): + # subsample training data + train_mask = mask + if (max_n is not None) and (np.sum(train_mask) > max_n): + n_train = int(max_n) + curr_train_idxs = np.nonzero(train_mask)[0] + rng = np.random.default_rng(seed=seed) + train_idxs_idx = rng.choice(len(curr_train_idxs), size=n_train, replace=False) + train_idxs = curr_train_idxs[train_idxs_idx] + train_mask = np.zeros_like(train_mask) + train_mask[train_idxs] = True + assert np.sum(train_mask) == n_train + return train_mask + + +class SequenceSampler: + + def __init__( + self, + replay_buffer: ReplayBuffer, + sequence_length: int, + pad_before: int = 0, + pad_after: int = 0, + keys=None, + key_first_k=dict(), + episode_mask: Optional[np.ndarray] = None, + ): + """ + key_first_k: dict str: int + Only take first k data from these keys (to improve perf) + """ + + super().__init__() + assert sequence_length >= 1 + if keys is None: + keys = list(replay_buffer.keys()) + + episode_ends = replay_buffer.episode_ends[:] + if episode_mask is None: + episode_mask = np.ones(episode_ends.shape, dtype=bool) + + if np.any(episode_mask): + indices = create_indices( + episode_ends, + sequence_length=sequence_length, + pad_before=pad_before, + pad_after=pad_after, + episode_mask=episode_mask, + ) + else: + indices = np.zeros((0, 4), dtype=np.int64) + + # (buffer_start_idx, buffer_end_idx, sample_start_idx, sample_end_idx) + self.indices = indices + self.keys = list(keys) # prevent OmegaConf list performance problem + self.sequence_length = sequence_length + self.replay_buffer = replay_buffer + self.key_first_k = key_first_k + + def __len__(self): + return len(self.indices) + + def sample_sequence(self, idx): + buffer_start_idx, buffer_end_idx, sample_start_idx, sample_end_idx = (self.indices[idx]) + result = dict() + for key in self.keys: + input_arr = self.replay_buffer[key] + # performance optimization, avoid small allocation if possible + if key not in self.key_first_k: + sample = input_arr[buffer_start_idx:buffer_end_idx] + else: + # performance optimization, only load used obs steps + n_data = buffer_end_idx - buffer_start_idx + k_data = min(self.key_first_k[key], n_data) + # fill value with Nan to catch bugs + # the non-loaded region should never be used + sample = np.full( + (n_data, ) + input_arr.shape[1:], + fill_value=np.nan, + dtype=input_arr.dtype, + ) + try: + sample[:k_data] = input_arr[buffer_start_idx:buffer_start_idx + k_data] + except Exception as e: + import pdb + + pdb.set_trace() + data = sample + if (sample_start_idx > 0) or (sample_end_idx < self.sequence_length): + data = np.zeros( + shape=(self.sequence_length, ) + input_arr.shape[1:], + dtype=input_arr.dtype, + ) + if sample_start_idx > 0: + data[:sample_start_idx] = sample[0] + if sample_end_idx < self.sequence_length: + data[sample_end_idx:] = sample[-1] + data[sample_start_idx:sample_end_idx] = sample + result[key] = data + return result diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/config/dp3.yaml b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/config/dp3.yaml new file mode 100644 index 00000000..059b6d27 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/config/dp3.yaml @@ -0,0 +1,147 @@ +defaults: + - task: adroit_hammer + +name: train_dp3 + +task_name: ${task.name} +shape_meta: ${task.shape_meta} +exp_name: "debug" + +horizon: 4 +n_obs_steps: 2 +n_action_steps: 4 +n_latency_steps: 0 +dataset_obs_steps: ${n_obs_steps} +keypoint_visible_rate: 1.0 +obs_as_global_cond: True + +policy: + _target_: diffusion_policy_3d.policy.dp3.DP3 + use_point_crop: true + condition_type: film + use_down_condition: true + use_mid_condition: true + use_up_condition: true + + diffusion_step_embed_dim: 128 + down_dims: + - 512 + - 1024 + - 2048 + crop_shape: + - 80 + - 80 + encoder_output_dim: 64 + horizon: ${horizon} + kernel_size: 5 + n_action_steps: ${n_action_steps} + n_groups: 8 + n_obs_steps: ${n_obs_steps} + + noise_scheduler: + _target_: diffusers.schedulers.scheduling_ddim.DDIMScheduler + num_train_timesteps: 100 + beta_start: 0.0001 + beta_end: 0.02 + beta_schedule: squaredcos_cap_v2 + clip_sample: True + set_alpha_to_one: True + steps_offset: 0 + prediction_type: sample + + + num_inference_steps: 10 + obs_as_global_cond: true + shape_meta: ${shape_meta} + + use_pc_color: false + pointnet_type: "pointnet" + + + pointcloud_encoder_cfg: + in_channels: 3 + out_channels: ${policy.encoder_output_dim} + use_layernorm: true + final_norm: layernorm # layernorm, none + normal_channel: false + + +ema: + _target_: diffusion_policy_3d.model.diffusion.ema_model.EMAModel + update_after_step: 0 + inv_gamma: 1.0 + power: 0.75 + min_value: 0.0 + max_value: 0.9999 + +dataloader: + batch_size: 128 + num_workers: 8 + shuffle: True + pin_memory: True + persistent_workers: False + +val_dataloader: + batch_size: 128 + num_workers: 8 + shuffle: False + pin_memory: True + persistent_workers: False + +optimizer: + _target_: torch.optim.AdamW + lr: 1.0e-4 + betas: [0.95, 0.999] + eps: 1.0e-8 + weight_decay: 1.0e-6 + +training: + device: "cuda:0" + seed: 42 + debug: False + resume: True + lr_scheduler: cosine + lr_warmup_steps: 500 + num_epochs: 3000 + gradient_accumulate_every: 1 + use_ema: True + rollout_every: 200 + checkpoint_every: 1 + val_every: 1 + sample_every: 5 + max_train_steps: null + max_val_steps: null + tqdm_interval_sec: 1.0 + +logging: + group: ${exp_name} + id: null + mode: online + name: ${training.seed} + project: dp3 + resume: true + tags: + - dp3 + +checkpoint: + save_ckpt: True # if True, save checkpoint every checkpoint_every + topk: + monitor_key: test_mean_score + mode: max + k: 1 + format_str: 'epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt' + save_last_ckpt: True # this only saves when save_ckpt is True + save_last_snapshot: False + +multi_run: + run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name} + wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name} + +hydra: + job: + override_dirname: ${name} + run: + dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name} + sweep: + dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name} + subdir: ${hydra.job.num} diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/config/robot_dp3.yaml b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/config/robot_dp3.yaml new file mode 100644 index 00000000..523a6071 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/config/robot_dp3.yaml @@ -0,0 +1,152 @@ +defaults: + - task: demo_task + +name: dp3 + +task_name: null +shape_meta: ${task.shape_meta} +exp_name: "debug" + +horizon: 8 +n_obs_steps: 3 +n_action_steps: 6 +n_latency_steps: 0 +dataset_obs_steps: ${n_obs_steps} +keypoint_visible_rate: 1.0 +obs_as_global_cond: True + +policy: + _target_: diffusion_policy_3d.policy.dp3.DP3 + use_point_crop: true + condition_type: film + use_down_condition: true + use_mid_condition: true + use_up_condition: true + + diffusion_step_embed_dim: 64 + down_dims: + - 512 + - 1024 + - 2048 + crop_shape: + - 80 + - 80 + encoder_output_dim: 128 # dual 128, raw 64 + horizon: ${horizon} + kernel_size: 5 + n_action_steps: ${n_action_steps} + n_groups: 8 + n_obs_steps: ${n_obs_steps} + + noise_scheduler: + _target_: diffusers.schedulers.scheduling_ddim.DDIMScheduler + num_train_timesteps: 100 + beta_start: 0.0001 + beta_end: 0.02 + beta_schedule: squaredcos_cap_v2 + clip_sample: True + set_alpha_to_one: True + steps_offset: 0 + prediction_type: sample + + + num_inference_steps: 10 + obs_as_global_cond: true + shape_meta: ${shape_meta} + + use_pc_color: false + pointnet_type: "pointnet" + + + pointcloud_encoder_cfg: + in_channels: 3 + out_channels: ${policy.encoder_output_dim} + use_layernorm: true + final_norm: layernorm # layernorm, none + normal_channel: false + + +ema: + _target_: diffusion_policy_3d.model.diffusion.ema_model.EMAModel + update_after_step: 0 + inv_gamma: 1.0 + power: 0.75 + min_value: 0.0 + max_value: 0.9999 + +dataloader: + batch_size: 256 + num_workers: 8 + shuffle: True + pin_memory: True + persistent_workers: False + +val_dataloader: + batch_size: 256 + num_workers: 8 + shuffle: False + pin_memory: True + persistent_workers: False + +optimizer: + _target_: torch.optim.AdamW + lr: 1.0e-4 + betas: [0.95, 0.999] + eps: 1.0e-8 + weight_decay: 1.0e-6 + +training: + device: "cuda:0" + seed: 42 + debug: False + resume: True + lr_scheduler: cosine + lr_warmup_steps: 500 + num_epochs: 3000 + gradient_accumulate_every: 1 + use_ema: True + rollout_every: 200 + checkpoint_every: 3000 + val_every: 50 + sample_every: 20 + max_train_steps: null + max_val_steps: null + tqdm_interval_sec: 1.0 + +logging: + group: ${exp_name} + id: null + mode: online + name: ${exp_name} + project: RoboTwin + resume: true + tags: + - RoboTwin + +checkpoint: + save_ckpt: False # if True, save checkpoint every checkpoint_every + topk: + monitor_key: test_mean_score + mode: max + k: 1 + format_str: 'epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt' + save_last_ckpt: True # this only saves when save_ckpt is True + save_last_snapshot: False + +hydra: + job: + override_dirname: ${name} + run: + dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name} + sweep: + dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name} + subdir: ${hydra.job.num} + +multi_run: + run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name} + wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name} + +checkpoint_num: 3000 +expert_data_num: 100 +raw_task_name: none +setting: none diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/config/task/demo_task.yaml b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/config/task/demo_task.yaml new file mode 100644 index 00000000..8a11cb74 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/config/task/demo_task.yaml @@ -0,0 +1,30 @@ +name: ${task_name}-${setting}-${expert_data_num} + +shape_meta: &shape_meta + # acceptable types: rgb, low_dim + obs: + point_cloud: + shape: [1024, 6] + type: point_cloud + agent_pos: + shape: [14] + type: low_dim + action: + shape: [14] + +env_runner: + _target_: diffusion_policy_3d.env_runner.robot_runner.RobotRunner + max_steps: 300 + n_obs_steps: ${n_obs_steps} + n_action_steps: ${n_action_steps} + task_name: robot + +dataset: + _target_: diffusion_policy_3d.dataset.robot_dataset.RobotDataset + zarr_path: ../../../data/${task.name}.zarr + horizon: ${horizon} + pad_before: ${eval:'${n_obs_steps}-1'} + pad_after: ${eval:'${n_action_steps}-1'} + seed: 0 + val_ratio: 0.02 + max_train_episodes: null diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/dataset/__init__.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/dataset/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/dataset/base_dataset.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/dataset/base_dataset.py new file mode 100644 index 00000000..6925323a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/dataset/base_dataset.py @@ -0,0 +1,30 @@ +from typing import Dict + +import torch +import torch.nn +from diffusion_policy_3d.model.common.normalizer import LinearNormalizer + + +class BaseDataset(torch.utils.data.Dataset): + + def get_validation_dataset(self) -> "BaseDataset": + # return an empty dataset by default + return BaseDataset() + + def get_normalizer(self, **kwargs) -> LinearNormalizer: + raise NotImplementedError() + + def get_all_actions(self) -> torch.Tensor: + raise NotImplementedError() + + def __len__(self) -> int: + return 0 + + def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]: + """ + output: + obs: + key: T, * + action: T, Da + """ + raise NotImplementedError() diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/dataset/robot_dataset.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/dataset/robot_dataset.py new file mode 100644 index 00000000..58e641a6 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/dataset/robot_dataset.py @@ -0,0 +1,107 @@ +import sys, os + +current_file_path = os.path.abspath(__file__) +parent_directory = os.path.dirname(current_file_path) +sys.path.append(os.path.join(parent_directory, '..')) +sys.path.append(os.path.join(parent_directory, '../..')) + +from typing import Dict +import torch +import numpy as np +import copy +from diffusion_policy_3d.common.pytorch_util import dict_apply +from diffusion_policy_3d.common.replay_buffer import ReplayBuffer +from diffusion_policy_3d.common.sampler import ( + SequenceSampler, + get_val_mask, + downsample_mask, +) +from diffusion_policy_3d.model.common.normalizer import ( + LinearNormalizer, + SingleFieldLinearNormalizer, +) +from diffusion_policy_3d.dataset.base_dataset import BaseDataset +import pdb + + +class RobotDataset(BaseDataset): + + def __init__( + self, + zarr_path, + horizon=1, + pad_before=0, + pad_after=0, + seed=42, + val_ratio=0.0, + max_train_episodes=None, + task_name=None, + ): + super().__init__() + self.task_name = task_name + current_file_path = os.path.abspath(__file__) + parent_directory = os.path.dirname(current_file_path) + zarr_path = os.path.join(parent_directory, zarr_path) + self.replay_buffer = ReplayBuffer.copy_from_path(zarr_path, keys=["state", "action", "point_cloud"]) # 'img' + val_mask = get_val_mask(n_episodes=self.replay_buffer.n_episodes, val_ratio=val_ratio, seed=seed) + train_mask = ~val_mask + train_mask = downsample_mask(mask=train_mask, max_n=max_train_episodes, seed=seed) + self.sampler = SequenceSampler( + replay_buffer=self.replay_buffer, + sequence_length=horizon, + pad_before=pad_before, + pad_after=pad_after, + episode_mask=train_mask, + ) + self.train_mask = train_mask + self.horizon = horizon + self.pad_before = pad_before + self.pad_after = pad_after + + def get_validation_dataset(self): + val_set = copy.copy(self) + val_set.sampler = SequenceSampler( + replay_buffer=self.replay_buffer, + sequence_length=self.horizon, + pad_before=self.pad_before, + pad_after=self.pad_after, + episode_mask=~self.train_mask, + ) + val_set.train_mask = ~self.train_mask + return val_set + + def get_normalizer(self, mode="limits", **kwargs): + data = { + "action": self.replay_buffer["action"], + "agent_pos": self.replay_buffer["state"][..., :], + "point_cloud": self.replay_buffer["point_cloud"], + } + normalizer = LinearNormalizer() + normalizer.fit(data=data, last_n_dims=1, mode=mode, **kwargs) + return normalizer + + def __len__(self) -> int: + return len(self.sampler) + + def _sample_to_data(self, sample): + agent_pos = sample["state"][ + :, + ].astype(np.float32) # (agent_posx2, block_posex3) + point_cloud = sample["point_cloud"][ + :, + ].astype(np.float32) # (T, 1024, 6) + + data = { + "obs": { + "point_cloud": point_cloud, # T, 1024, 6 + "agent_pos": agent_pos, # T, D_pos + }, + "action": sample["action"].astype(np.float32), # T, D_action + } + return data + + def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]: + sample = self.sampler.sample_sequence(idx) + data = self._sample_to_data(sample) + torch_data = dict_apply(data, torch.from_numpy) + return torch_data diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/env_runner/base_runner.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/env_runner/base_runner.py new file mode 100644 index 00000000..446d27ec --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/env_runner/base_runner.py @@ -0,0 +1,11 @@ +from typing import Dict +from diffusion_policy_3d.policy.base_policy import BasePolicy + + +class BaseRunner: + + def __init__(self, output_dir): + self.output_dir = output_dir + + def run(self, policy: BasePolicy) -> Dict: + raise NotImplementedError() diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/env_runner/robot_runner.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/env_runner/robot_runner.py new file mode 100644 index 00000000..674918d8 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/env_runner/robot_runner.py @@ -0,0 +1,112 @@ +import wandb +import numpy as np +import torch +import tqdm + +from diffusion_policy_3d.policy.base_policy import BasePolicy +from diffusion_policy_3d.common.pytorch_util import dict_apply +from diffusion_policy_3d.env_runner.base_runner import BaseRunner +import diffusion_policy_3d.common.logger_util as logger_util +from termcolor import cprint +import pdb +from queue import deque + + +class RobotRunner(BaseRunner): + + def __init__( + self, + output_dir=None, + eval_episodes=20, + max_steps=200, + n_obs_steps=8, + n_action_steps=8, + fps=10, + crf=22, + tqdm_interval_sec=5.0, + task_name=None + ): + super().__init__(output_dir) + self.task_name = task_name + + steps_per_render = max(10 // fps, 1) + + self.eval_episodes = eval_episodes + self.fps = fps + self.crf = crf + self.n_obs_steps = n_obs_steps + self.n_action_steps = n_action_steps + self.max_steps = max_steps + self.tqdm_interval_sec = tqdm_interval_sec + + self.logger_util_test = logger_util.LargestKRecorder(K=3) + self.logger_util_test10 = logger_util.LargestKRecorder(K=5) + self.obs = deque(maxlen=n_obs_steps + 1) + self.env = None + + def stack_last_n_obs(self, all_obs, n_steps): + assert len(all_obs) > 0 + all_obs = list(all_obs) + if isinstance(all_obs[0], np.ndarray): + result = np.zeros((n_steps, ) + all_obs[-1].shape, dtype=all_obs[-1].dtype) + start_idx = -min(n_steps, len(all_obs)) + result[start_idx:] = np.array(all_obs[start_idx:]) + if n_steps > len(all_obs): + # pad + result[:start_idx] = result[start_idx] + elif isinstance(all_obs[0], torch.Tensor): + result = torch.zeros((n_steps, ) + all_obs[-1].shape, dtype=all_obs[-1].dtype) + start_idx = -min(n_steps, len(all_obs)) + result[start_idx:] = torch.stack(all_obs[start_idx:]) + if n_steps > len(all_obs): + # pad + result[:start_idx] = result[start_idx] + else: + raise RuntimeError(f"Unsupported obs type {type(all_obs[0])}") + return result + + def reset_obs(self): + self.obs.clear() + + def update_obs(self, current_obs): + self.obs.append(current_obs) + + def get_n_steps_obs(self): + assert len(self.obs) > 0, "no observation is recorded, please update obs first" + + result = dict() + for key in self.obs[0].keys(): + result[key] = self.stack_last_n_obs([obs[key] for obs in self.obs], self.n_obs_steps) + + return result + + def get_action(self, policy: BasePolicy, observaton=None) -> bool: + device, dtype = policy.device, policy.dtype + if observaton is not None: + self.obs.append(observaton) # update + obs = self.get_n_steps_obs() + + # create obs dict + np_obs_dict = dict(obs) + # device transfer + obs_dict = dict_apply(np_obs_dict, lambda x: torch.from_numpy(x).to(device=device)) + # run policy + with torch.no_grad(): + obs_dict_input = {} # flush unused keys + obs_dict_input["point_cloud"] = obs_dict["point_cloud"].unsqueeze(0) + obs_dict_input["agent_pos"] = obs_dict["agent_pos"].unsqueeze(0) + + action_dict = policy.predict_action(obs_dict_input) + + # device_transfer + np_action_dict = dict_apply(action_dict, lambda x: x.detach().to("cpu").numpy()) + action = np_action_dict["action"].squeeze(0) + return action + + def run(self, policy: BasePolicy): + pass + + +if __name__ == "__main__": + test = RobotRunner("./") + print("ready") diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/common/dict_of_tensor_mixin.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/common/dict_of_tensor_mixin.py new file mode 100644 index 00000000..358da9fe --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/common/dict_of_tensor_mixin.py @@ -0,0 +1,50 @@ +import torch +import torch.nn as nn + + +class DictOfTensorMixin(nn.Module): + + def __init__(self, params_dict=None): + super().__init__() + if params_dict is None: + params_dict = nn.ParameterDict() + self.params_dict = params_dict + + @property + def device(self): + return next(iter(self.parameters())).device + + def _load_from_state_dict( + self, + state_dict, + prefix, + local_metadata, + strict, + missing_keys, + unexpected_keys, + error_msgs, + ): + + def dfs_add(dest, keys, value: torch.Tensor): + if len(keys) == 1: + dest[keys[0]] = value + return + + if keys[0] not in dest: + dest[keys[0]] = nn.ParameterDict() + dfs_add(dest[keys[0]], keys[1:], value) + + def load_dict(state_dict, prefix): + out_dict = nn.ParameterDict() + for key, value in state_dict.items(): + value: torch.Tensor + if key.startswith(prefix): + param_keys = key[len(prefix):].split(".")[1:] + # if len(param_keys) == 0: + # import pdb; pdb.set_trace() + dfs_add(out_dict, param_keys, value.clone()) + return out_dict + + self.params_dict = load_dict(state_dict, prefix + "params_dict") + self.params_dict.requires_grad_(False) + return diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/common/lr_scheduler.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/common/lr_scheduler.py new file mode 100644 index 00000000..c97f3052 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/common/lr_scheduler.py @@ -0,0 +1,55 @@ +from diffusers.optimization import ( + Union, + SchedulerType, + Optional, + Optimizer, + TYPE_TO_SCHEDULER_FUNCTION, +) + + +def get_scheduler( + name: Union[str, SchedulerType], + optimizer: Optimizer, + num_warmup_steps: Optional[int] = None, + num_training_steps: Optional[int] = None, + **kwargs, +): + """ + Added kwargs vs diffuser's original implementation + + Unified API to get any scheduler from its name. + + Args: + name (`str` or `SchedulerType`): + The name of the scheduler to use. + optimizer (`torch.optim.Optimizer`): + The optimizer that will be used during training. + num_warmup_steps (`int`, *optional*): + The number of warmup steps to do. This is not required by all schedulers (hence the argument being + optional), the function will raise an error if it's unset and the scheduler type requires it. + num_training_steps (`int``, *optional*): + The number of training steps to do. This is not required by all schedulers (hence the argument being + optional), the function will raise an error if it's unset and the scheduler type requires it. + """ + name = SchedulerType(name) + schedule_func = TYPE_TO_SCHEDULER_FUNCTION[name] + if name == SchedulerType.CONSTANT: + return schedule_func(optimizer, **kwargs) + + # All other schedulers require `num_warmup_steps` + if num_warmup_steps is None: + raise ValueError(f"{name} requires `num_warmup_steps`, please provide that argument.") + + if name == SchedulerType.CONSTANT_WITH_WARMUP: + return schedule_func(optimizer, num_warmup_steps=num_warmup_steps, **kwargs) + + # All other schedulers require `num_training_steps` + if num_training_steps is None: + raise ValueError(f"{name} requires `num_training_steps`, please provide that argument.") + + return schedule_func( + optimizer, + num_warmup_steps=num_warmup_steps, + num_training_steps=num_training_steps, + **kwargs, + ) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/common/module_attr_mixin.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/common/module_attr_mixin.py new file mode 100644 index 00000000..e33efe29 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/common/module_attr_mixin.py @@ -0,0 +1,16 @@ +import torch.nn as nn + + +class ModuleAttrMixin(nn.Module): + + def __init__(self): + super().__init__() + self._dummy_variable = nn.Parameter() + + @property + def device(self): + return next(iter(self.parameters())).device + + @property + def dtype(self): + return next(iter(self.parameters())).dtype diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/common/normalizer.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/common/normalizer.py new file mode 100644 index 00000000..2233c0e6 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/common/normalizer.py @@ -0,0 +1,367 @@ +from typing import Union, Dict + +import unittest +import zarr +import numpy as np +import torch +import torch.nn as nn +from diffusion_policy_3d.common.pytorch_util import dict_apply +from diffusion_policy_3d.model.common.dict_of_tensor_mixin import DictOfTensorMixin + + +class LinearNormalizer(DictOfTensorMixin): + avaliable_modes = ["limits", "gaussian"] + + @torch.no_grad() + def fit( + self, + data: Union[Dict, torch.Tensor, np.ndarray, zarr.Array], + last_n_dims=1, + dtype=torch.float32, + mode="limits", + output_max=1.0, + output_min=-1.0, + range_eps=1e-4, + fit_offset=True, + ): + if isinstance(data, dict): + for key, value in data.items(): + self.params_dict[key] = _fit( + value, + last_n_dims=last_n_dims, + dtype=dtype, + mode=mode, + output_max=output_max, + output_min=output_min, + range_eps=range_eps, + fit_offset=fit_offset, + ) + else: + self.params_dict["_default"] = _fit( + data, + last_n_dims=last_n_dims, + dtype=dtype, + mode=mode, + output_max=output_max, + output_min=output_min, + range_eps=range_eps, + fit_offset=fit_offset, + ) + + def __call__(self, x: Union[Dict, torch.Tensor, np.ndarray]) -> torch.Tensor: + return self.normalize(x) + + def __getitem__(self, key: str): + return SingleFieldLinearNormalizer(self.params_dict[key]) + + def __setitem__(self, key: str, value: "SingleFieldLinearNormalizer"): + self.params_dict[key] = value.params_dict + + def _normalize_impl(self, x, forward=True): + if isinstance(x, dict): + result = dict() + for key, value in x.items(): + params = self.params_dict[key] + result[key] = _normalize(value, params, forward=forward) + return result + else: + if "_default" not in self.params_dict: + raise RuntimeError("Not initialized") + params = self.params_dict["_default"] + return _normalize(x, params, forward=forward) + + def normalize(self, x: Union[Dict, torch.Tensor, np.ndarray]) -> torch.Tensor: + return self._normalize_impl(x, forward=True) + + def unnormalize(self, x: Union[Dict, torch.Tensor, np.ndarray]) -> torch.Tensor: + return self._normalize_impl(x, forward=False) + + def get_input_stats(self) -> Dict: + if len(self.params_dict) == 0: + raise RuntimeError("Not initialized") + if len(self.params_dict) == 1 and "_default" in self.params_dict: + return self.params_dict["_default"]["input_stats"] + + result = dict() + for key, value in self.params_dict.items(): + if key != "_default": + result[key] = value["input_stats"] + return result + + def get_output_stats(self, key="_default"): + input_stats = self.get_input_stats() + if "min" in input_stats: + # no dict + return dict_apply(input_stats, self.normalize) + + result = dict() + for key, group in input_stats.items(): + this_dict = dict() + for name, value in group.items(): + this_dict[name] = self.normalize({key: value})[key] + result[key] = this_dict + return result + + +class SingleFieldLinearNormalizer(DictOfTensorMixin): + avaliable_modes = ["limits", "gaussian"] + + @torch.no_grad() + def fit( + self, + data: Union[torch.Tensor, np.ndarray, zarr.Array], + last_n_dims=1, + dtype=torch.float32, + mode="limits", + output_max=1.0, + output_min=-1.0, + range_eps=1e-4, + fit_offset=True, + ): + self.params_dict = _fit( + data, + last_n_dims=last_n_dims, + dtype=dtype, + mode=mode, + output_max=output_max, + output_min=output_min, + range_eps=range_eps, + fit_offset=fit_offset, + ) + + @classmethod + def create_fit(cls, data: Union[torch.Tensor, np.ndarray, zarr.Array], **kwargs): + obj = cls() + obj.fit(data, **kwargs) + return obj + + @classmethod + def create_manual( + cls, + scale: Union[torch.Tensor, np.ndarray], + offset: Union[torch.Tensor, np.ndarray], + input_stats_dict: Dict[str, Union[torch.Tensor, np.ndarray]], + ): + + def to_tensor(x): + if not isinstance(x, torch.Tensor): + x = torch.from_numpy(x) + x = x.flatten() + return x + + # check + for x in [offset] + list(input_stats_dict.values()): + assert x.shape == scale.shape + assert x.dtype == scale.dtype + + params_dict = nn.ParameterDict({ + "scale": to_tensor(scale), + "offset": to_tensor(offset), + "input_stats": nn.ParameterDict(dict_apply(input_stats_dict, to_tensor)), + }) + return cls(params_dict) + + @classmethod + def create_identity(cls, dtype=torch.float32): + scale = torch.tensor([1], dtype=dtype) + offset = torch.tensor([0], dtype=dtype) + input_stats_dict = { + "min": torch.tensor([-1], dtype=dtype), + "max": torch.tensor([1], dtype=dtype), + "mean": torch.tensor([0], dtype=dtype), + "std": torch.tensor([1], dtype=dtype), + } + return cls.create_manual(scale, offset, input_stats_dict) + + def normalize(self, x: Union[torch.Tensor, np.ndarray]) -> torch.Tensor: + return _normalize(x, self.params_dict, forward=True) + + def unnormalize(self, x: Union[torch.Tensor, np.ndarray]) -> torch.Tensor: + return _normalize(x, self.params_dict, forward=False) + + def get_input_stats(self): + return self.params_dict["input_stats"] + + def get_output_stats(self): + return dict_apply(self.params_dict["input_stats"], self.normalize) + + def __call__(self, x: Union[torch.Tensor, np.ndarray]) -> torch.Tensor: + return self.normalize(x) + + +def _fit( + data: Union[torch.Tensor, np.ndarray, zarr.Array], + last_n_dims=1, + dtype=torch.float32, + mode="limits", + output_max=1.0, + output_min=-1.0, + range_eps=1e-4, + fit_offset=True, +): + assert mode in ["limits", "gaussian"] + assert last_n_dims >= 0 + assert output_max > output_min + + # convert data to torch and type + if isinstance(data, zarr.Array): + data = data[:] + if isinstance(data, np.ndarray): + data = torch.from_numpy(data) + if dtype is not None: + data = data.type(dtype) + + # convert shape + dim = 1 + if last_n_dims > 0: + dim = np.prod(data.shape[-last_n_dims:]) + data = data.reshape(-1, dim) + + # compute input stats min max mean std + input_min, _ = data.min(axis=0) + input_max, _ = data.max(axis=0) + input_mean = data.mean(axis=0) + input_std = data.std(axis=0) + + # compute scale and offset + if mode == "limits": + if fit_offset: + # unit scale + input_range = input_max - input_min + ignore_dim = input_range < range_eps + input_range[ignore_dim] = output_max - output_min + scale = (output_max - output_min) / input_range + offset = output_min - scale * input_min + offset[ignore_dim] = (output_max + output_min) / 2 - input_min[ignore_dim] + # ignore dims scaled to mean of output max and min + else: + # use this when data is pre-zero-centered. + assert output_max > 0 + assert output_min < 0 + # unit abs + output_abs = min(abs(output_min), abs(output_max)) + input_abs = torch.maximum(torch.abs(input_min), torch.abs(input_max)) + ignore_dim = input_abs < range_eps + input_abs[ignore_dim] = output_abs + # don't scale constant channels + scale = output_abs / input_abs + offset = torch.zeros_like(input_mean) + elif mode == "gaussian": + ignore_dim = input_std < range_eps + scale = input_std.clone() + scale[ignore_dim] = 1 + scale = 1 / scale + + if fit_offset: + offset = -input_mean * scale + else: + offset = torch.zeros_like(input_mean) + + # save + this_params = nn.ParameterDict({ + "scale": + scale, + "offset": + offset, + "input_stats": + nn.ParameterDict({ + "min": input_min, + "max": input_max, + "mean": input_mean, + "std": input_std, + }), + }) + for p in this_params.parameters(): + p.requires_grad_(False) + return this_params + + +def _normalize(x, params, forward=True): + assert "scale" in params + if isinstance(x, np.ndarray): + x = torch.from_numpy(x) + scale = params["scale"] + offset = params["offset"] + x = x.to(device=scale.device, dtype=scale.dtype) + src_shape = x.shape + x = x.reshape(-1, scale.shape[0]) + if forward: + x = x * scale + offset + else: + x = (x - offset) / scale + x = x.reshape(src_shape) + return x + + +def test(): + data = torch.zeros((100, 10, 9, 2)).uniform_() + data[..., 0, 0] = 0 + + normalizer = SingleFieldLinearNormalizer() + normalizer.fit(data, mode="limits", last_n_dims=2) + datan = normalizer.normalize(data) + assert datan.shape == data.shape + assert np.allclose(datan.max(), 1.0) + assert np.allclose(datan.min(), -1.0) + dataun = normalizer.unnormalize(datan) + assert torch.allclose(data, dataun, atol=1e-7) + + input_stats = normalizer.get_input_stats() + output_stats = normalizer.get_output_stats() + + normalizer = SingleFieldLinearNormalizer() + normalizer.fit(data, mode="limits", last_n_dims=1, fit_offset=False) + datan = normalizer.normalize(data) + assert datan.shape == data.shape + assert np.allclose(datan.max(), 1.0, atol=1e-3) + assert np.allclose(datan.min(), 0.0, atol=1e-3) + dataun = normalizer.unnormalize(datan) + assert torch.allclose(data, dataun, atol=1e-7) + + data = torch.zeros((100, 10, 9, 2)).uniform_() + normalizer = SingleFieldLinearNormalizer() + normalizer.fit(data, mode="gaussian", last_n_dims=0) + datan = normalizer.normalize(data) + assert datan.shape == data.shape + assert np.allclose(datan.mean(), 0.0, atol=1e-3) + assert np.allclose(datan.std(), 1.0, atol=1e-3) + dataun = normalizer.unnormalize(datan) + assert torch.allclose(data, dataun, atol=1e-7) + + # dict + data = torch.zeros((100, 10, 9, 2)).uniform_() + data[..., 0, 0] = 0 + + normalizer = LinearNormalizer() + normalizer.fit(data, mode="limits", last_n_dims=2) + datan = normalizer.normalize(data) + assert datan.shape == data.shape + assert np.allclose(datan.max(), 1.0) + assert np.allclose(datan.min(), -1.0) + dataun = normalizer.unnormalize(datan) + assert torch.allclose(data, dataun, atol=1e-7) + + input_stats = normalizer.get_input_stats() + output_stats = normalizer.get_output_stats() + + data = { + "obs": torch.zeros((1000, 128, 9, 2)).uniform_() * 512, + "action": torch.zeros((1000, 128, 2)).uniform_() * 512, + } + normalizer = LinearNormalizer() + normalizer.fit(data) + datan = normalizer.normalize(data) + dataun = normalizer.unnormalize(datan) + for key in data: + assert torch.allclose(data[key], dataun[key], atol=1e-4) + + input_stats = normalizer.get_input_stats() + output_stats = normalizer.get_output_stats() + + state_dict = normalizer.state_dict() + n = LinearNormalizer() + n.load_state_dict(state_dict) + datan = n.normalize(data) + dataun = n.unnormalize(datan) + for key in data: + assert torch.allclose(data[key], dataun[key], atol=1e-4) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/common/shape_util.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/common/shape_util.py new file mode 100644 index 00000000..2445d8ad --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/common/shape_util.py @@ -0,0 +1,22 @@ +from typing import Dict, List, Tuple, Callable +import torch +import torch.nn as nn + + +def get_module_device(m: nn.Module): + device = torch.device("cpu") + try: + param = next(iter(m.parameters())) + device = param.device + except StopIteration: + pass + return device + + +@torch.no_grad() +def get_output_shape(input_shape: Tuple[int], net: Callable[[torch.Tensor], torch.Tensor]): + device = get_module_device(net) + test_input = torch.zeros((1, ) + tuple(input_shape), device=device) + test_output = net(test_input) + output_shape = tuple(test_output.shape[1:]) + return output_shape diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/common/tensor_util.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/common/tensor_util.py new file mode 100644 index 00000000..f0fc7dd1 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/common/tensor_util.py @@ -0,0 +1,972 @@ +""" +A collection of utilities for working with nested tensor structures consisting +of numpy arrays and torch tensors. +""" + +import collections +import numpy as np +import torch + + +def recursive_dict_list_tuple_apply(x, type_func_dict): + """ + Recursively apply functions to a nested dictionary or list or tuple, given a dictionary of + {data_type: function_to_apply}. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + type_func_dict (dict): a mapping from data types to the functions to be + applied for each data type. + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + assert list not in type_func_dict + assert tuple not in type_func_dict + assert dict not in type_func_dict + + if isinstance(x, (dict, collections.OrderedDict)): + new_x = (collections.OrderedDict() if isinstance(x, collections.OrderedDict) else dict()) + for k, v in x.items(): + new_x[k] = recursive_dict_list_tuple_apply(v, type_func_dict) + return new_x + elif isinstance(x, (list, tuple)): + ret = [recursive_dict_list_tuple_apply(v, type_func_dict) for v in x] + if isinstance(x, tuple): + ret = tuple(ret) + return ret + else: + for t, f in type_func_dict.items(): + if isinstance(x, t): + return f(x) + else: + raise NotImplementedError("Cannot handle data type %s" % str(type(x))) + + +def map_tensor(x, func): + """ + Apply function @func to torch.Tensor objects in a nested dictionary or + list or tuple. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + func (function): function to apply to each tensor + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: func, + type(None): lambda x: x, + }, + ) + + +def map_ndarray(x, func): + """ + Apply function @func to np.ndarray objects in a nested dictionary or + list or tuple. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + func (function): function to apply to each array + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + np.ndarray: func, + type(None): lambda x: x, + }, + ) + + +def map_tensor_ndarray(x, tensor_func, ndarray_func): + """ + Apply function @tensor_func to torch.Tensor objects and @ndarray_func to + np.ndarray objects in a nested dictionary or list or tuple. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + tensor_func (function): function to apply to each tensor + ndarray_Func (function): function to apply to each array + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: tensor_func, + np.ndarray: ndarray_func, + type(None): lambda x: x, + }, + ) + + +def clone(x): + """ + Clones all torch tensors and numpy arrays in nested dictionary or list + or tuple and returns a new nested structure. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: lambda x: x.clone(), + np.ndarray: lambda x: x.copy(), + type(None): lambda x: x, + }, + ) + + +def detach(x): + """ + Detaches all torch tensors in nested dictionary or list + or tuple and returns a new nested structure. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: lambda x: x.detach(), + }, + ) + + +def to_batch(x): + """ + Introduces a leading batch dimension of 1 for all torch tensors and numpy + arrays in nested dictionary or list or tuple and returns a new nested structure. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: lambda x: x[None, ...], + np.ndarray: lambda x: x[None, ...], + type(None): lambda x: x, + }, + ) + + +def to_sequence(x): + """ + Introduces a time dimension of 1 at dimension 1 for all torch tensors and numpy + arrays in nested dictionary or list or tuple and returns a new nested structure. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: lambda x: x[:, None, ...], + np.ndarray: lambda x: x[:, None, ...], + type(None): lambda x: x, + }, + ) + + +def index_at_time(x, ind): + """ + Indexes all torch tensors and numpy arrays in dimension 1 with index @ind in + nested dictionary or list or tuple and returns a new nested structure. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + ind (int): index + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: lambda x: x[:, ind, ...], + np.ndarray: lambda x: x[:, ind, ...], + type(None): lambda x: x, + }, + ) + + +def unsqueeze(x, dim): + """ + Adds dimension of size 1 at dimension @dim in all torch tensors and numpy arrays + in nested dictionary or list or tuple and returns a new nested structure. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + dim (int): dimension + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: lambda x: x.unsqueeze(dim=dim), + np.ndarray: lambda x: np.expand_dims(x, axis=dim), + type(None): lambda x: x, + }, + ) + + +def contiguous(x): + """ + Makes all torch tensors and numpy arrays contiguous in nested dictionary or + list or tuple and returns a new nested structure. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: lambda x: x.contiguous(), + np.ndarray: lambda x: np.ascontiguousarray(x), + type(None): lambda x: x, + }, + ) + + +def to_device(x, device): + """ + Sends all torch tensors in nested dictionary or list or tuple to device + @device, and returns a new nested structure. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + device (torch.Device): device to send tensors to + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: lambda x, d=device: x.to(d), + type(None): lambda x: x, + }, + ) + + +def to_tensor(x): + """ + Converts all numpy arrays in nested dictionary or list or tuple to + torch tensors (and leaves existing torch Tensors as-is), and returns + a new nested structure. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: lambda x: x, + np.ndarray: lambda x: torch.from_numpy(x), + type(None): lambda x: x, + }, + ) + + +def to_numpy(x): + """ + Converts all torch tensors in nested dictionary or list or tuple to + numpy (and leaves existing numpy arrays as-is), and returns + a new nested structure. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + + def f(tensor): + if tensor.is_cuda: + return tensor.detach().cpu().numpy() + else: + return tensor.detach().numpy() + + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: f, + np.ndarray: lambda x: x, + type(None): lambda x: x, + }, + ) + + +def to_list(x): + """ + Converts all torch tensors and numpy arrays in nested dictionary or list + or tuple to a list, and returns a new nested structure. Useful for + json encoding. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + + def f(tensor): + if tensor.is_cuda: + return tensor.detach().cpu().numpy().tolist() + else: + return tensor.detach().numpy().tolist() + + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: f, + np.ndarray: lambda x: x.tolist(), + type(None): lambda x: x, + }, + ) + + +def to_float(x): + """ + Converts all torch tensors and numpy arrays in nested dictionary or list + or tuple to float type entries, and returns a new nested structure. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: lambda x: x.float(), + np.ndarray: lambda x: x.astype(np.float32), + type(None): lambda x: x, + }, + ) + + +def to_uint8(x): + """ + Converts all torch tensors and numpy arrays in nested dictionary or list + or tuple to uint8 type entries, and returns a new nested structure. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: lambda x: x.byte(), + np.ndarray: lambda x: x.astype(np.uint8), + type(None): lambda x: x, + }, + ) + + +def to_torch(x, device): + """ + Converts all numpy arrays and torch tensors in nested dictionary or list or tuple to + torch tensors on device @device and returns a new nested structure. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + device (torch.Device): device to send tensors to + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return to_device(to_float(to_tensor(x)), device) + + +def to_one_hot_single(tensor, num_class): + """ + Convert tensor to one-hot representation, assuming a certain number of total class labels. + + Args: + tensor (torch.Tensor): tensor containing integer labels + num_class (int): number of classes + + Returns: + x (torch.Tensor): tensor containing one-hot representation of labels + """ + x = torch.zeros(tensor.size() + (num_class, )).to(tensor.device) + x.scatter_(-1, tensor.unsqueeze(-1), 1) + return x + + +def to_one_hot(tensor, num_class): + """ + Convert all tensors in nested dictionary or list or tuple to one-hot representation, + assuming a certain number of total class labels. + + Args: + tensor (dict or list or tuple): a possibly nested dictionary or list or tuple + num_class (int): number of classes + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return map_tensor(tensor, func=lambda x, nc=num_class: to_one_hot_single(x, nc)) + + +def flatten_single(x, begin_axis=1): + """ + Flatten a tensor in all dimensions from @begin_axis onwards. + + Args: + x (torch.Tensor): tensor to flatten + begin_axis (int): which axis to flatten from + + Returns: + y (torch.Tensor): flattened tensor + """ + fixed_size = x.size()[:begin_axis] + _s = list(fixed_size) + [-1] + return x.reshape(*_s) + + +def flatten(x, begin_axis=1): + """ + Flatten all tensors in nested dictionary or list or tuple, from @begin_axis onwards. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + begin_axis (int): which axis to flatten from + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: lambda x, b=begin_axis: flatten_single(x, begin_axis=b), + }, + ) + + +def reshape_dimensions_single(x, begin_axis, end_axis, target_dims): + """ + Reshape selected dimensions in a tensor to a target dimension. + + Args: + x (torch.Tensor): tensor to reshape + begin_axis (int): begin dimension + end_axis (int): end dimension + target_dims (tuple or list): target shape for the range of dimensions + (@begin_axis, @end_axis) + + Returns: + y (torch.Tensor): reshaped tensor + """ + assert begin_axis <= end_axis + assert begin_axis >= 0 + assert end_axis < len(x.shape) + assert isinstance(target_dims, (tuple, list)) + s = x.shape + final_s = [] + for i in range(len(s)): + if i == begin_axis: + final_s.extend(target_dims) + elif i < begin_axis or i > end_axis: + final_s.append(s[i]) + return x.reshape(*final_s) + + +def reshape_dimensions(x, begin_axis, end_axis, target_dims): + """ + Reshape selected dimensions for all tensors in nested dictionary or list or tuple + to a target dimension. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + begin_axis (int): begin dimension + end_axis (int): end dimension + target_dims (tuple or list): target shape for the range of dimensions + (@begin_axis, @end_axis) + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: + lambda x, b=begin_axis, e=end_axis, t=target_dims: reshape_dimensions_single( + x, begin_axis=b, end_axis=e, target_dims=t), + np.ndarray: + lambda x, b=begin_axis, e=end_axis, t=target_dims: reshape_dimensions_single( + x, begin_axis=b, end_axis=e, target_dims=t), + type(None): + lambda x: x, + }, + ) + + +def join_dimensions(x, begin_axis, end_axis): + """ + Joins all dimensions between dimensions (@begin_axis, @end_axis) into a flat dimension, for + all tensors in nested dictionary or list or tuple. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + begin_axis (int): begin dimension + end_axis (int): end dimension + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: + lambda x, b=begin_axis, e=end_axis: reshape_dimensions_single(x, begin_axis=b, end_axis=e, target_dims=[-1] + ), + np.ndarray: + lambda x, b=begin_axis, e=end_axis: reshape_dimensions_single(x, begin_axis=b, end_axis=e, target_dims=[-1] + ), + type(None): + lambda x: x, + }, + ) + + +def expand_at_single(x, size, dim): + """ + Expand a tensor at a single dimension @dim by @size + + Args: + x (torch.Tensor): input tensor + size (int): size to expand + dim (int): dimension to expand + + Returns: + y (torch.Tensor): expanded tensor + """ + assert dim < x.ndimension() + assert x.shape[dim] == 1 + expand_dims = [-1] * x.ndimension() + expand_dims[dim] = size + return x.expand(*expand_dims) + + +def expand_at(x, size, dim): + """ + Expand all tensors in nested dictionary or list or tuple at a single + dimension @dim by @size. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + size (int): size to expand + dim (int): dimension to expand + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return map_tensor(x, lambda t, s=size, d=dim: expand_at_single(t, s, d)) + + +def unsqueeze_expand_at(x, size, dim): + """ + Unsqueeze and expand a tensor at a dimension @dim by @size. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + size (int): size to expand + dim (int): dimension to unsqueeze and expand + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + x = unsqueeze(x, dim) + return expand_at(x, size, dim) + + +def repeat_by_expand_at(x, repeats, dim): + """ + Repeat a dimension by combining expand and reshape operations. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + repeats (int): number of times to repeat the target dimension + dim (int): dimension to repeat on + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + x = unsqueeze_expand_at(x, repeats, dim + 1) + return join_dimensions(x, dim, dim + 1) + + +def named_reduce_single(x, reduction, dim): + """ + Reduce tensor at a dimension by named reduction functions. + + Args: + x (torch.Tensor): tensor to be reduced + reduction (str): one of ["sum", "max", "mean", "flatten"] + dim (int): dimension to be reduced (or begin axis for flatten) + + Returns: + y (torch.Tensor): reduced tensor + """ + assert x.ndimension() > dim + assert reduction in ["sum", "max", "mean", "flatten"] + if reduction == "flatten": + x = flatten(x, begin_axis=dim) + elif reduction == "max": + x = torch.max(x, dim=dim)[0] # [B, D] + elif reduction == "sum": + x = torch.sum(x, dim=dim) + else: + x = torch.mean(x, dim=dim) + return x + + +def named_reduce(x, reduction, dim): + """ + Reduces all tensors in nested dictionary or list or tuple at a dimension + using a named reduction function. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + reduction (str): one of ["sum", "max", "mean", "flatten"] + dim (int): dimension to be reduced (or begin axis for flatten) + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return map_tensor(x, func=lambda t, r=reduction, d=dim: named_reduce_single(t, r, d)) + + +def gather_along_dim_with_dim_single(x, target_dim, source_dim, indices): + """ + This function indexes out a target dimension of a tensor in a structured way, + by allowing a different value to be selected for each member of a flat index + tensor (@indices) corresponding to a source dimension. This can be interpreted + as moving along the source dimension, using the corresponding index value + in @indices to select values for all other dimensions outside of the + source and target dimensions. A common use case is to gather values + in target dimension 1 for each batch member (target dimension 0). + + Args: + x (torch.Tensor): tensor to gather values for + target_dim (int): dimension to gather values along + source_dim (int): dimension to hold constant and use for gathering values + from the other dimensions + indices (torch.Tensor): flat index tensor with same shape as tensor @x along + @source_dim + + Returns: + y (torch.Tensor): gathered tensor, with dimension @target_dim indexed out + """ + assert len(indices.shape) == 1 + assert x.shape[source_dim] == indices.shape[0] + + # unsqueeze in all dimensions except the source dimension + new_shape = [1] * x.ndimension() + new_shape[source_dim] = -1 + indices = indices.reshape(*new_shape) + + # repeat in all dimensions - but preserve shape of source dimension, + # and make sure target_dimension has singleton dimension + expand_shape = list(x.shape) + expand_shape[source_dim] = -1 + expand_shape[target_dim] = 1 + indices = indices.expand(*expand_shape) + + out = x.gather(dim=target_dim, index=indices) + return out.squeeze(target_dim) + + +def gather_along_dim_with_dim(x, target_dim, source_dim, indices): + """ + Apply @gather_along_dim_with_dim_single to all tensors in a nested + dictionary or list or tuple. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + target_dim (int): dimension to gather values along + source_dim (int): dimension to hold constant and use for gathering values + from the other dimensions + indices (torch.Tensor): flat index tensor with same shape as tensor @x along + @source_dim + + Returns: + y (dict or list or tuple): new nested dict-list-tuple + """ + return map_tensor( + x, + lambda y, t=target_dim, s=source_dim, i=indices: gather_along_dim_with_dim_single(y, t, s, i), + ) + + +def gather_sequence_single(seq, indices): + """ + Given a tensor with leading dimensions [B, T, ...], gather an element from each sequence in + the batch given an index for each sequence. + + Args: + seq (torch.Tensor): tensor with leading dimensions [B, T, ...] + indices (torch.Tensor): tensor indices of shape [B] + + Return: + y (torch.Tensor): indexed tensor of shape [B, ....] + """ + return gather_along_dim_with_dim_single(seq, target_dim=1, source_dim=0, indices=indices) + + +def gather_sequence(seq, indices): + """ + Given a nested dictionary or list or tuple, gathers an element from each sequence of the batch + for tensors with leading dimensions [B, T, ...]. + + Args: + seq (dict or list or tuple): a possibly nested dictionary or list or tuple with tensors + of leading dimensions [B, T, ...] + indices (torch.Tensor): tensor indices of shape [B] + + Returns: + y (dict or list or tuple): new nested dict-list-tuple with tensors of shape [B, ...] + """ + return gather_along_dim_with_dim(seq, target_dim=1, source_dim=0, indices=indices) + + +def pad_sequence_single(seq, padding, batched=False, pad_same=True, pad_values=None): + """ + Pad input tensor or array @seq in the time dimension (dimension 1). + + Args: + seq (np.ndarray or torch.Tensor): sequence to be padded + padding (tuple): begin and end padding, e.g. [1, 1] pads both begin and end of the sequence by 1 + batched (bool): if sequence has the batch dimension + pad_same (bool): if pad by duplicating + pad_values (scalar or (ndarray, Tensor)): values to be padded if not pad_same + + Returns: + padded sequence (np.ndarray or torch.Tensor) + """ + assert isinstance(seq, (np.ndarray, torch.Tensor)) + assert pad_same or pad_values is not None + if pad_values is not None: + assert isinstance(pad_values, float) + repeat_func = np.repeat if isinstance(seq, np.ndarray) else torch.repeat_interleave + concat_func = np.concatenate if isinstance(seq, np.ndarray) else torch.cat + ones_like_func = np.ones_like if isinstance(seq, np.ndarray) else torch.ones_like + seq_dim = 1 if batched else 0 + + begin_pad = [] + end_pad = [] + + if padding[0] > 0: + pad = seq[[0]] if pad_same else ones_like_func(seq[[0]]) * pad_values + begin_pad.append(repeat_func(pad, padding[0], seq_dim)) + if padding[1] > 0: + pad = seq[[-1]] if pad_same else ones_like_func(seq[[-1]]) * pad_values + end_pad.append(repeat_func(pad, padding[1], seq_dim)) + + return concat_func(begin_pad + [seq] + end_pad, seq_dim) + + +def pad_sequence(seq, padding, batched=False, pad_same=True, pad_values=None): + """ + Pad a nested dictionary or list or tuple of sequence tensors in the time dimension (dimension 1). + + Args: + seq (dict or list or tuple): a possibly nested dictionary or list or tuple with tensors + of leading dimensions [B, T, ...] + padding (tuple): begin and end padding, e.g. [1, 1] pads both begin and end of the sequence by 1 + batched (bool): if sequence has the batch dimension + pad_same (bool): if pad by duplicating + pad_values (scalar or (ndarray, Tensor)): values to be padded if not pad_same + + Returns: + padded sequence (dict or list or tuple) + """ + return recursive_dict_list_tuple_apply( + seq, + { + torch.Tensor: + lambda x, p=padding, b=batched, ps=pad_same, pv=pad_values: pad_sequence_single(x, p, b, ps, pv), + np.ndarray: + lambda x, p=padding, b=batched, ps=pad_same, pv=pad_values: pad_sequence_single(x, p, b, ps, pv), + type(None): lambda x: x, + }, + ) + + +def assert_size_at_dim_single(x, size, dim, msg): + """ + Ensure that array or tensor @x has size @size in dim @dim. + + Args: + x (np.ndarray or torch.Tensor): input array or tensor + size (int): size that tensors should have at @dim + dim (int): dimension to check + msg (str): text to display if assertion fails + """ + assert x.shape[dim] == size, msg + + +def assert_size_at_dim(x, size, dim, msg): + """ + Ensure that arrays and tensors in nested dictionary or list or tuple have + size @size in dim @dim. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + size (int): size that tensors should have at @dim + dim (int): dimension to check + """ + map_tensor(x, lambda t, s=size, d=dim, m=msg: assert_size_at_dim_single(t, s, d, m)) + + +def get_shape(x): + """ + Get all shapes of arrays and tensors in nested dictionary or list or tuple. + + Args: + x (dict or list or tuple): a possibly nested dictionary or list or tuple + + Returns: + y (dict or list or tuple): new nested dict-list-tuple that contains each array or + tensor's shape + """ + return recursive_dict_list_tuple_apply( + x, + { + torch.Tensor: lambda x: x.shape, + np.ndarray: lambda x: x.shape, + type(None): lambda x: x, + }, + ) + + +def list_of_flat_dict_to_dict_of_list(list_of_dict): + """ + Helper function to go from a list of flat dictionaries to a dictionary of lists. + By "flat" we mean that none of the values are dictionaries, but are numpy arrays, + floats, etc. + + Args: + list_of_dict (list): list of flat dictionaries + + Returns: + dict_of_list (dict): dictionary of lists + """ + assert isinstance(list_of_dict, list) + dic = collections.OrderedDict() + for i in range(len(list_of_dict)): + for k in list_of_dict[i]: + if k not in dic: + dic[k] = [] + dic[k].append(list_of_dict[i][k]) + return dic + + +def flatten_nested_dict_list(d, parent_key="", sep="_", item_key=""): + """ + Flatten a nested dict or list to a list. + + For example, given a dict + { + a: 1 + b: { + c: 2 + } + c: 3 + } + + the function would return [(a, 1), (b_c, 2), (c, 3)] + + Args: + d (dict, list): a nested dict or list to be flattened + parent_key (str): recursion helper + sep (str): separator for nesting keys + item_key (str): recursion helper + Returns: + list: a list of (key, value) tuples + """ + items = [] + if isinstance(d, (tuple, list)): + new_key = parent_key + sep + item_key if len(parent_key) > 0 else item_key + for i, v in enumerate(d): + items.extend(flatten_nested_dict_list(v, new_key, sep=sep, item_key=str(i))) + return items + elif isinstance(d, dict): + new_key = parent_key + sep + item_key if len(parent_key) > 0 else item_key + for k, v in d.items(): + assert isinstance(k, str) + items.extend(flatten_nested_dict_list(v, new_key, sep=sep, item_key=k)) + return items + else: + new_key = parent_key + sep + item_key if len(parent_key) > 0 else item_key + return [(new_key, d)] + + +def time_distributed(inputs, op, activation=None, inputs_as_kwargs=False, inputs_as_args=False, **kwargs): + """ + Apply function @op to all tensors in nested dictionary or list or tuple @inputs in both the + batch (B) and time (T) dimension, where the tensors are expected to have shape [B, T, ...]. + Will do this by reshaping tensors to [B * T, ...], passing through the op, and then reshaping + outputs to [B, T, ...]. + + Args: + inputs (list or tuple or dict): a possibly nested dictionary or list or tuple with tensors + of leading dimensions [B, T, ...] + op: a layer op that accepts inputs + activation: activation to apply at the output + inputs_as_kwargs (bool): whether to feed input as a kwargs dict to the op + inputs_as_args (bool) whether to feed input as a args list to the op + kwargs (dict): other kwargs to supply to the op + + Returns: + outputs (dict or list or tuple): new nested dict-list-tuple with tensors of leading dimension [B, T]. + """ + batch_size, seq_len = flatten_nested_dict_list(inputs)[0][1].shape[:2] + inputs = join_dimensions(inputs, 0, 1) + if inputs_as_kwargs: + outputs = op(**inputs, **kwargs) + elif inputs_as_args: + outputs = op(*inputs, **kwargs) + else: + outputs = op(inputs, **kwargs) + + if activation is not None: + outputs = map_tensor(outputs, activation) + outputs = reshape_dimensions(outputs, begin_axis=0, end_axis=0, target_dims=(batch_size, seq_len)) + return outputs diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/diffusion/conditional_unet1d.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/diffusion/conditional_unet1d.py new file mode 100644 index 00000000..2fe260e5 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/diffusion/conditional_unet1d.py @@ -0,0 +1,373 @@ +from typing import Union +import logging +import torch +import torch.nn as nn +import torch.nn.functional as F +import einops +from einops.layers.torch import Rearrange +from termcolor import cprint +from diffusion_policy_3d.model.diffusion.conv1d_components import ( + Downsample1d, + Upsample1d, + Conv1dBlock, +) +from diffusion_policy_3d.model.diffusion.positional_embedding import SinusoidalPosEmb + +logger = logging.getLogger(__name__) + + +class CrossAttention(nn.Module): + + def __init__(self, in_dim, cond_dim, out_dim): + super().__init__() + self.query_proj = nn.Linear(in_dim, out_dim) + self.key_proj = nn.Linear(cond_dim, out_dim) + self.value_proj = nn.Linear(cond_dim, out_dim) + + def forward(self, x, cond): + # x: [batch_size, t_act, in_dim] + # cond: [batch_size, t_obs, cond_dim] + + # Project x and cond to query, key, and value + query = self.query_proj(x) # [batch_size, horizon, out_dim] + key = self.key_proj(cond) # [batch_size, horizon, out_dim] + value = self.value_proj(cond) # [batch_size, horizon, out_dim] + + # Compute attention + attn_weights = torch.matmul(query, key.transpose(-2, -1)) # [batch_size, horizon, horizon] + attn_weights = F.softmax(attn_weights, dim=-1) + + # Apply attention + attn_output = torch.matmul(attn_weights, value) # [batch_size, horizon, out_dim] + + return attn_output + + +class ConditionalResidualBlock1D(nn.Module): + + def __init__( + self, + in_channels, + out_channels, + cond_dim, + kernel_size=3, + n_groups=8, + condition_type="film", + ): + super().__init__() + + self.blocks = nn.ModuleList([ + Conv1dBlock(in_channels, out_channels, kernel_size, n_groups=n_groups), + Conv1dBlock(out_channels, out_channels, kernel_size, n_groups=n_groups), + ]) + + self.condition_type = condition_type + + cond_channels = out_channels + if condition_type == "film": # FiLM modulation https://arxiv.org/abs/1709.07871 + # predicts per-channel scale and bias + cond_channels = out_channels * 2 + self.cond_encoder = nn.Sequential( + nn.Mish(), + nn.Linear(cond_dim, cond_channels), + Rearrange("batch t -> batch t 1"), + ) + elif condition_type == "add": + self.cond_encoder = nn.Sequential( + nn.Mish(), + nn.Linear(cond_dim, out_channels), + Rearrange("batch t -> batch t 1"), + ) + elif condition_type == "cross_attention_add": + self.cond_encoder = CrossAttention(in_channels, cond_dim, out_channels) + elif condition_type == "cross_attention_film": + cond_channels = out_channels * 2 + self.cond_encoder = CrossAttention(in_channels, cond_dim, cond_channels) + elif condition_type == "mlp_film": + cond_channels = out_channels * 2 + self.cond_encoder = nn.Sequential( + nn.Mish(), + nn.Linear(cond_dim, cond_dim), + nn.Mish(), + nn.Linear(cond_dim, cond_channels), + Rearrange("batch t -> batch t 1"), + ) + else: + raise NotImplementedError(f"condition_type {condition_type} not implemented") + + self.out_channels = out_channels + # make sure dimensions compatible + self.residual_conv = (nn.Conv1d(in_channels, out_channels, 1) if in_channels != out_channels else nn.Identity()) + + def forward(self, x, cond=None): + """ + x : [ batch_size x in_channels x horizon ] + cond : [ batch_size x cond_dim] + + returns: + out : [ batch_size x out_channels x horizon ] + """ + out = self.blocks[0](x) + if cond is not None: + if self.condition_type == "film": + embed = self.cond_encoder(cond) + embed = embed.reshape(embed.shape[0], 2, self.out_channels, 1) + scale = embed[:, 0, ...] + bias = embed[:, 1, ...] + out = scale * out + bias + elif self.condition_type == "add": + embed = self.cond_encoder(cond) + out = out + embed + elif self.condition_type == "cross_attention_add": + embed = self.cond_encoder(x.permute(0, 2, 1), cond) + embed = embed.permute(0, 2, 1) # [batch_size, out_channels, horizon] + out = out + embed + elif self.condition_type == "cross_attention_film": + embed = self.cond_encoder(x.permute(0, 2, 1), cond) + embed = embed.permute(0, 2, 1) + embed = embed.reshape(embed.shape[0], 2, self.out_channels, -1) + scale = embed[:, 0, ...] + bias = embed[:, 1, ...] + out = scale * out + bias + elif self.condition_type == "mlp_film": + embed = self.cond_encoder(cond) + embed = embed.reshape(embed.shape[0], 2, self.out_channels, -1) + scale = embed[:, 0, ...] + bias = embed[:, 1, ...] + out = scale * out + bias + else: + raise NotImplementedError(f"condition_type {self.condition_type} not implemented") + out = self.blocks[1](out) + out = out + self.residual_conv(x) + return out + + +class ConditionalUnet1D(nn.Module): + + def __init__( + self, + input_dim, + local_cond_dim=None, + global_cond_dim=None, + diffusion_step_embed_dim=256, + down_dims=[256, 512, 1024], + kernel_size=3, + n_groups=8, + condition_type="film", + use_down_condition=True, + use_mid_condition=True, + use_up_condition=True, + ): + super().__init__() + self.condition_type = condition_type + + self.use_down_condition = use_down_condition + self.use_mid_condition = use_mid_condition + self.use_up_condition = use_up_condition + + all_dims = [input_dim] + list(down_dims) + start_dim = down_dims[0] + + dsed = diffusion_step_embed_dim + diffusion_step_encoder = nn.Sequential( + SinusoidalPosEmb(dsed), + nn.Linear(dsed, dsed * 4), + nn.Mish(), + nn.Linear(dsed * 4, dsed), + ) + cond_dim = dsed + if global_cond_dim is not None: + cond_dim += global_cond_dim + + in_out = list(zip(all_dims[:-1], all_dims[1:])) + + local_cond_encoder = None + if local_cond_dim is not None: + _, dim_out = in_out[0] + dim_in = local_cond_dim + local_cond_encoder = nn.ModuleList([ + # down encoder + ConditionalResidualBlock1D( + dim_in, + dim_out, + cond_dim=cond_dim, + kernel_size=kernel_size, + n_groups=n_groups, + condition_type=condition_type, + ), + # up encoder + ConditionalResidualBlock1D( + dim_in, + dim_out, + cond_dim=cond_dim, + kernel_size=kernel_size, + n_groups=n_groups, + condition_type=condition_type, + ), + ]) + + mid_dim = all_dims[-1] + self.mid_modules = nn.ModuleList([ + ConditionalResidualBlock1D( + mid_dim, + mid_dim, + cond_dim=cond_dim, + kernel_size=kernel_size, + n_groups=n_groups, + condition_type=condition_type, + ), + ConditionalResidualBlock1D( + mid_dim, + mid_dim, + cond_dim=cond_dim, + kernel_size=kernel_size, + n_groups=n_groups, + condition_type=condition_type, + ), + ]) + + down_modules = nn.ModuleList([]) + for ind, (dim_in, dim_out) in enumerate(in_out): + is_last = ind >= (len(in_out) - 1) + down_modules.append( + nn.ModuleList([ + ConditionalResidualBlock1D( + dim_in, + dim_out, + cond_dim=cond_dim, + kernel_size=kernel_size, + n_groups=n_groups, + condition_type=condition_type, + ), + ConditionalResidualBlock1D( + dim_out, + dim_out, + cond_dim=cond_dim, + kernel_size=kernel_size, + n_groups=n_groups, + condition_type=condition_type, + ), + Downsample1d(dim_out) if not is_last else nn.Identity(), + ])) + + up_modules = nn.ModuleList([]) + for ind, (dim_in, dim_out) in enumerate(reversed(in_out[1:])): + is_last = ind >= (len(in_out) - 1) + up_modules.append( + nn.ModuleList([ + ConditionalResidualBlock1D( + dim_out * 2, + dim_in, + cond_dim=cond_dim, + kernel_size=kernel_size, + n_groups=n_groups, + condition_type=condition_type, + ), + ConditionalResidualBlock1D( + dim_in, + dim_in, + cond_dim=cond_dim, + kernel_size=kernel_size, + n_groups=n_groups, + condition_type=condition_type, + ), + Upsample1d(dim_in) if not is_last else nn.Identity(), + ])) + + final_conv = nn.Sequential( + Conv1dBlock(start_dim, start_dim, kernel_size=kernel_size), + nn.Conv1d(start_dim, input_dim, 1), + ) + + self.diffusion_step_encoder = diffusion_step_encoder + self.local_cond_encoder = local_cond_encoder + self.up_modules = up_modules + self.down_modules = down_modules + self.final_conv = final_conv + + logger.info("number of parameters: %e", sum(p.numel() for p in self.parameters())) + + def forward( + self, + sample: torch.Tensor, + timestep: Union[torch.Tensor, float, int], + local_cond=None, + global_cond=None, + **kwargs, + ): + """ + x: (B,T,input_dim) + timestep: (B,) or int, diffusion step + local_cond: (B,T,local_cond_dim) + global_cond: (B,global_cond_dim) + output: (B,T,input_dim) + """ + sample = einops.rearrange(sample, "b h t -> b t h") + + # 1. time + timesteps = timestep + if not torch.is_tensor(timesteps): + # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can + timesteps = torch.tensor([timesteps], dtype=torch.long, device=sample.device) + elif torch.is_tensor(timesteps) and len(timesteps.shape) == 0: + timesteps = timesteps[None].to(sample.device) + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timesteps = timesteps.expand(sample.shape[0]) + + timestep_embed = self.diffusion_step_encoder(timesteps) + if global_cond is not None: + if self.condition_type == "cross_attention": + timestep_embed = timestep_embed.unsqueeze(1).expand(-1, global_cond.shape[1], -1) + global_feature = torch.cat([timestep_embed, global_cond], axis=-1) + + # encode local features + h_local = list() + if local_cond is not None: + local_cond = einops.rearrange(local_cond, "b h t -> b t h") + resnet, resnet2 = self.local_cond_encoder + x = resnet(local_cond, global_feature) + h_local.append(x) + x = resnet2(local_cond, global_feature) + h_local.append(x) + + x = sample + h = [] + for idx, (resnet, resnet2, downsample) in enumerate(self.down_modules): + if self.use_down_condition: + x = resnet(x, global_feature) + if idx == 0 and len(h_local) > 0: + x = x + h_local[0] + x = resnet2(x, global_feature) + else: + x = resnet(x) + if idx == 0 and len(h_local) > 0: + x = x + h_local[0] + x = resnet2(x) + h.append(x) + x = downsample(x) + + for mid_module in self.mid_modules: + if self.use_mid_condition: + x = mid_module(x, global_feature) + else: + x = mid_module(x) + + for idx, (resnet, resnet2, upsample) in enumerate(self.up_modules): + x = torch.cat((x, h.pop()), dim=1) + if self.use_up_condition: + x = resnet(x, global_feature) + if idx == len(self.up_modules) and len(h_local) > 0: + x = x + h_local[1] + x = resnet2(x, global_feature) + else: + x = resnet(x) + if idx == len(self.up_modules) and len(h_local) > 0: + x = x + h_local[1] + x = resnet2(x) + x = upsample(x) + + x = self.final_conv(x) + + x = einops.rearrange(x, "b t h -> b h t") + + return x diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/diffusion/conv1d_components.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/diffusion/conv1d_components.py new file mode 100644 index 00000000..163ed05e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/diffusion/conv1d_components.py @@ -0,0 +1,51 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +# from einops.layers.torch import Rearrange + + +class Downsample1d(nn.Module): + + def __init__(self, dim): + super().__init__() + self.conv = nn.Conv1d(dim, dim, 3, 2, 1) + + def forward(self, x): + return self.conv(x) + + +class Upsample1d(nn.Module): + + def __init__(self, dim): + super().__init__() + self.conv = nn.ConvTranspose1d(dim, dim, 4, 2, 1) + + def forward(self, x): + return self.conv(x) + + +class Conv1dBlock(nn.Module): + """ + Conv1d --> GroupNorm --> Mish + """ + + def __init__(self, inp_channels, out_channels, kernel_size, n_groups=8): + super().__init__() + + self.block = nn.Sequential( + nn.Conv1d(inp_channels, out_channels, kernel_size, padding=kernel_size // 2), + # Rearrange('batch channels horizon -> batch channels 1 horizon'), + nn.GroupNorm(n_groups, out_channels), + # Rearrange('batch channels 1 horizon -> batch channels horizon'), + nn.Mish(), + ) + + def forward(self, x): + return self.block(x) + + +def test(): + cb = Conv1dBlock(256, 128, kernel_size=3) + x = torch.zeros((1, 256, 16)) + o = cb(x) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/diffusion/ema_model.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/diffusion/ema_model.py new file mode 100644 index 00000000..c6835f75 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/diffusion/ema_model.py @@ -0,0 +1,89 @@ +import copy +import torch +from torch.nn.modules.batchnorm import _BatchNorm + + +class EMAModel: + """ + Exponential Moving Average of models weights + """ + + def __init__( + self, + model, + update_after_step=0, + inv_gamma=1.0, + power=2 / 3, + min_value=0.0, + max_value=0.9999, + ): + """ + @crowsonkb's notes on EMA Warmup: + If gamma=1 and power=1, implements a simple average. gamma=1, power=2/3 are good values for models you plan + to train for a million or more steps (reaches decay factor 0.999 at 31.6K steps, 0.9999 at 1M steps), + gamma=1, power=3/4 for models you plan to train for less (reaches decay factor 0.999 at 10K steps, 0.9999 + at 215.4k steps). + Args: + inv_gamma (float): Inverse multiplicative factor of EMA warmup. Default: 1. + power (float): Exponential factor of EMA warmup. Default: 2/3. + min_value (float): The minimum EMA decay rate. Default: 0. + """ + + self.averaged_model = model + self.averaged_model.eval() + self.averaged_model.requires_grad_(False) + + self.update_after_step = update_after_step + self.inv_gamma = inv_gamma + self.power = power + self.min_value = min_value + self.max_value = max_value + + self.decay = 0.0 + self.optimization_step = 0 + + def get_decay(self, optimization_step): + """ + Compute the decay factor for the exponential moving average. + """ + step = max(0, optimization_step - self.update_after_step - 1) + value = 1 - (1 + step / self.inv_gamma)**-self.power + + if step <= 0: + return 0.0 + + return max(self.min_value, min(value, self.max_value)) + + @torch.no_grad() + def step(self, new_model): + self.decay = self.get_decay(self.optimization_step) + + # old_all_dataptrs = set() + # for param in new_model.parameters(): + # data_ptr = param.data_ptr() + # if data_ptr != 0: + # old_all_dataptrs.add(data_ptr) + + all_dataptrs = set() + for module, ema_module in zip(new_model.modules(), self.averaged_model.modules()): + for param, ema_param in zip(module.parameters(recurse=False), ema_module.parameters(recurse=False)): + # iterative over immediate parameters only. + if isinstance(param, dict): + raise RuntimeError("Dict parameter not supported") + + # data_ptr = param.data_ptr() + # if data_ptr != 0: + # all_dataptrs.add(data_ptr) + + if isinstance(module, _BatchNorm): + # skip batchnorms + ema_param.copy_(param.to(dtype=ema_param.dtype).data) + elif not param.requires_grad: + ema_param.copy_(param.to(dtype=ema_param.dtype).data) + else: + ema_param.mul_(self.decay) + ema_param.add_(param.data.to(dtype=ema_param.dtype), alpha=1 - self.decay) + + # verify that iterating over module and then parameters is identical to parameters recursively. + # assert old_all_dataptrs == all_dataptrs + self.optimization_step += 1 diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/diffusion/mask_generator.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/diffusion/mask_generator.py new file mode 100644 index 00000000..a0b92ac3 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/diffusion/mask_generator.py @@ -0,0 +1,225 @@ +from typing import Sequence, Optional +import torch +from torch import nn +from diffusion_policy_3d.model.common.module_attr_mixin import ModuleAttrMixin + + +def get_intersection_slice_mask(shape: tuple, dim_slices: Sequence[slice], device: Optional[torch.device] = None): + assert len(shape) == len(dim_slices) + mask = torch.zeros(size=shape, dtype=torch.bool, device=device) + mask[dim_slices] = True + return mask + + +def get_union_slice_mask(shape: tuple, dim_slices: Sequence[slice], device: Optional[torch.device] = None): + assert len(shape) == len(dim_slices) + mask = torch.zeros(size=shape, dtype=torch.bool, device=device) + for i in range(len(dim_slices)): + this_slices = [slice(None)] * len(shape) + this_slices[i] = dim_slices[i] + mask[this_slices] = True + return mask + + +class DummyMaskGenerator(ModuleAttrMixin): + + def __init__(self): + super().__init__() + + @torch.no_grad() + def forward(self, shape): + device = self.device + mask = torch.ones(size=shape, dtype=torch.bool, device=device) + return mask + + +class LowdimMaskGenerator(ModuleAttrMixin): + + def __init__( + self, + action_dim, + obs_dim, + # obs mask setup + max_n_obs_steps=2, + fix_obs_steps=True, + # action mask + action_visible=False, + ): + super().__init__() + self.action_dim = action_dim + self.obs_dim = obs_dim + self.max_n_obs_steps = max_n_obs_steps + self.fix_obs_steps = fix_obs_steps + self.action_visible = action_visible + + @torch.no_grad() + def forward(self, shape, seed=None): + device = self.device + B, T, D = shape + assert D == (self.action_dim + self.obs_dim) + + # create all tensors on this device + rng = torch.Generator(device=device) + if seed is not None: + rng = rng.manual_seed(seed) + + # generate dim mask + dim_mask = torch.zeros(size=shape, dtype=torch.bool, device=device) + is_action_dim = dim_mask.clone() + is_action_dim[..., :self.action_dim] = True + is_obs_dim = ~is_action_dim + + # generate obs mask + if self.fix_obs_steps: + obs_steps = torch.full((B, ), fill_value=self.max_n_obs_steps, device=device) + else: + obs_steps = torch.randint( + low=1, + high=self.max_n_obs_steps + 1, + size=(B, ), + generator=rng, + device=device, + ) + + steps = torch.arange(0, T, device=device).reshape(1, T).expand(B, T) + obs_mask = (steps.T < obs_steps).T.reshape(B, T, 1).expand(B, T, D) + obs_mask = obs_mask & is_obs_dim + + # generate action mask + if self.action_visible: + action_steps = torch.maximum( + obs_steps - 1, + torch.tensor(0, dtype=obs_steps.dtype, device=obs_steps.device), + ) + action_mask = (steps.T < action_steps).T.reshape(B, T, 1).expand(B, T, D) + action_mask = action_mask & is_action_dim + + mask = obs_mask + if self.action_visible: + mask = mask | action_mask + + return mask + + +class KeypointMaskGenerator(ModuleAttrMixin): + + def __init__( + self, + # dimensions + action_dim, + keypoint_dim, + # obs mask setup + max_n_obs_steps=2, + fix_obs_steps=True, + # keypoint mask setup + keypoint_visible_rate=0.7, + time_independent=False, + # action mask + action_visible=False, + context_dim=0, # dim for context + n_context_steps=1, + ): + super().__init__() + self.action_dim = action_dim + self.keypoint_dim = keypoint_dim + self.context_dim = context_dim + self.max_n_obs_steps = max_n_obs_steps + self.fix_obs_steps = fix_obs_steps + self.keypoint_visible_rate = keypoint_visible_rate + self.time_independent = time_independent + self.action_visible = action_visible + self.n_context_steps = n_context_steps + + @torch.no_grad() + def forward(self, shape, seed=None): + device = self.device + B, T, D = shape + all_keypoint_dims = D - self.action_dim - self.context_dim + n_keypoints = all_keypoint_dims // self.keypoint_dim + + # create all tensors on this device + rng = torch.Generator(device=device) + if seed is not None: + rng = rng.manual_seed(seed) + + # generate dim mask + dim_mask = torch.zeros(size=shape, dtype=torch.bool, device=device) + is_action_dim = dim_mask.clone() + is_action_dim[..., :self.action_dim] = True + is_context_dim = dim_mask.clone() + if self.context_dim > 0: + is_context_dim[..., -self.context_dim:] = True + is_obs_dim = ~(is_action_dim | is_context_dim) + # assumption trajectory=cat([action, keypoints, context], dim=-1) + + # generate obs mask + if self.fix_obs_steps: + obs_steps = torch.full((B, ), fill_value=self.max_n_obs_steps, device=device) + else: + obs_steps = torch.randint( + low=1, + high=self.max_n_obs_steps + 1, + size=(B, ), + generator=rng, + device=device, + ) + + steps = torch.arange(0, T, device=device).reshape(1, T).expand(B, T) + obs_mask = (steps.T < obs_steps).T.reshape(B, T, 1).expand(B, T, D) + obs_mask = obs_mask & is_obs_dim + + # generate action mask + if self.action_visible: + action_steps = torch.maximum( + obs_steps - 1, + torch.tensor(0, dtype=obs_steps.dtype, device=obs_steps.device), + ) + action_mask = (steps.T < action_steps).T.reshape(B, T, 1).expand(B, T, D) + action_mask = action_mask & is_action_dim + + # generate keypoint mask + if self.time_independent: + visible_kps = (torch.rand(size=(B, T, n_keypoints), generator=rng, device=device) + < self.keypoint_visible_rate) + visible_dims = torch.repeat_interleave(visible_kps, repeats=self.keypoint_dim, dim=-1) + visible_dims_mask = torch.cat( + [ + torch.ones((B, T, self.action_dim), dtype=torch.bool, device=device), + visible_dims, + torch.ones((B, T, self.context_dim), dtype=torch.bool, device=device), + ], + axis=-1, + ) + keypoint_mask = visible_dims_mask + else: + visible_kps = (torch.rand(size=(B, n_keypoints), generator=rng, device=device) < self.keypoint_visible_rate) + visible_dims = torch.repeat_interleave(visible_kps, repeats=self.keypoint_dim, dim=-1) + visible_dims_mask = torch.cat( + [ + torch.ones((B, self.action_dim), dtype=torch.bool, device=device), + visible_dims, + torch.ones((B, self.context_dim), dtype=torch.bool, device=device), + ], + axis=-1, + ) + keypoint_mask = visible_dims_mask.reshape(B, 1, D).expand(B, T, D) + keypoint_mask = keypoint_mask & is_obs_dim + + # generate context mask + context_mask = is_context_dim.clone() + context_mask[:, self.n_context_steps:, :] = False + + mask = obs_mask & keypoint_mask + if self.action_visible: + mask = mask | action_mask + if self.context_dim > 0: + mask = mask | context_mask + + return mask + + +def test(): + # kmg = KeypointMaskGenerator(2,2, random_obs_steps=True) + # self = KeypointMaskGenerator(2,2,context_dim=2, action_visible=True) + # self = KeypointMaskGenerator(2,2,context_dim=0, action_visible=True) + self = LowdimMaskGenerator(2, 20, max_n_obs_steps=3, action_visible=True) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/diffusion/positional_embedding.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/diffusion/positional_embedding.py new file mode 100644 index 00000000..1b1d646d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/diffusion/positional_embedding.py @@ -0,0 +1,19 @@ +import math +import torch +import torch.nn as nn + + +class SinusoidalPosEmb(nn.Module): + + def __init__(self, dim): + super().__init__() + self.dim = dim + + def forward(self, x): + device = x.device + half_dim = self.dim // 2 + emb = math.log(10000) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, device=device) * -emb) + emb = x[:, None] * emb[None, :] + emb = torch.cat((emb.sin(), emb.cos()), dim=-1) + return emb diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/diffusion/simple_conditional_unet1d.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/diffusion/simple_conditional_unet1d.py new file mode 100644 index 00000000..4fff65ac --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/diffusion/simple_conditional_unet1d.py @@ -0,0 +1,323 @@ +from typing import Union +import logging +import torch +import torch.nn as nn +import einops +from einops.layers.torch import Rearrange +from termcolor import cprint +from diffusion_policy_3d.model.diffusion.conv1d_components import ( + Downsample1d, + Upsample1d, + Conv1dBlock, +) +from diffusion_policy_3d.model.diffusion.positional_embedding import SinusoidalPosEmb +from diffusion_policy_3d.common.model_util import print_params + +logger = logging.getLogger(__name__) + + +class ConditionalResidualBlock1D(nn.Module): + + def __init__( + self, + in_channels, + out_channels, + cond_dim, + kernel_size=3, + n_groups=8, + condition_type="film", + ): + super().__init__() + + self.blocks = nn.ModuleList([ + Conv1dBlock(in_channels, out_channels, kernel_size, n_groups=n_groups), + Conv1dBlock(out_channels, out_channels, kernel_size, n_groups=n_groups), + ]) + + self.condition_type = condition_type + + cond_channels = out_channels + if condition_type == "film": # FiLM modulation https://arxiv.org/abs/1709.07871 + # predicts per-channel scale and bias + cond_channels = out_channels * 2 + self.cond_encoder = nn.Sequential( + nn.Mish(), + nn.Linear(cond_dim, cond_channels), + Rearrange("batch t -> batch t 1"), + ) + elif condition_type == "add": + self.cond_encoder = nn.Sequential( + nn.Mish(), + nn.Linear(cond_dim, out_channels), + Rearrange("batch t -> batch t 1"), + ) + elif condition_type == "mlp_film": + cond_channels = out_channels * 2 + self.cond_encoder = nn.Sequential( + nn.Mish(), + nn.Linear(cond_dim, cond_dim), + nn.Mish(), + nn.Linear(cond_dim, cond_channels), + Rearrange("batch t -> batch t 1"), + ) + else: + raise NotImplementedError(f"condition_type {condition_type} not implemented") + + self.out_channels = out_channels + # make sure dimensions compatible + self.residual_conv = (nn.Conv1d(in_channels, out_channels, 1) if in_channels != out_channels else nn.Identity()) + + def forward(self, x, cond=None): + """ + x : [ batch_size x in_channels x horizon ] + cond : [ batch_size x cond_dim] + + returns: + out : [ batch_size x out_channels x horizon ] + """ + out = self.blocks[0](x) + if cond is not None: + if self.condition_type == "film": + embed = self.cond_encoder(cond) + embed = embed.reshape(embed.shape[0], 2, self.out_channels, 1) + scale = embed[:, 0, ...] + bias = embed[:, 1, ...] + out = scale * out + bias + elif self.condition_type == "add": + embed = self.cond_encoder(cond) + out = out + embed + elif self.condition_type == "mlp_film": + embed = self.cond_encoder(cond) + embed = embed.reshape(embed.shape[0], 2, self.out_channels, -1) + scale = embed[:, 0, ...] + bias = embed[:, 1, ...] + out = scale * out + bias + else: + raise NotImplementedError(f"condition_type {self.condition_type} not implemented") + out = self.blocks[1](out) + out = out + self.residual_conv(x) + return out + + +class ConditionalUnet1D(nn.Module): + + def __init__( + self, + input_dim, + local_cond_dim=None, + global_cond_dim=None, + diffusion_step_embed_dim=256, + down_dims=[256, 512, 1024], + kernel_size=3, + n_groups=8, + condition_type="film", + use_down_condition=True, + use_mid_condition=True, + use_up_condition=True, + ): + super().__init__() + self.condition_type = condition_type + + self.use_down_condition = use_down_condition + self.use_mid_condition = use_mid_condition + self.use_up_condition = use_up_condition + + all_dims = [input_dim] + list(down_dims) + start_dim = down_dims[0] + + dsed = diffusion_step_embed_dim + diffusion_step_encoder = nn.Sequential( + SinusoidalPosEmb(dsed), + nn.Linear(dsed, dsed * 4), + nn.Mish(), + nn.Linear(dsed * 4, dsed), + ) + cond_dim = dsed + if global_cond_dim is not None: + cond_dim += global_cond_dim + + in_out = list(zip(all_dims[:-1], all_dims[1:])) + + local_cond_encoder = None + if local_cond_dim is not None: + _, dim_out = in_out[0] + dim_in = local_cond_dim + local_cond_encoder = nn.ModuleList([ + # down encoder + ConditionalResidualBlock1D( + dim_in, + dim_out, + cond_dim=cond_dim, + kernel_size=kernel_size, + n_groups=n_groups, + condition_type=condition_type, + ), + # up encoder + ConditionalResidualBlock1D( + dim_in, + dim_out, + cond_dim=cond_dim, + kernel_size=kernel_size, + n_groups=n_groups, + condition_type=condition_type, + ), + ]) + + mid_dim = all_dims[-1] + self.mid_modules = nn.ModuleList([ + ConditionalResidualBlock1D( + mid_dim, + mid_dim, + cond_dim=cond_dim, + kernel_size=kernel_size, + n_groups=n_groups, + condition_type=condition_type, + ), + # ConditionalResidualBlock1D( + # mid_dim, mid_dim, cond_dim=cond_dim, + # kernel_size=kernel_size, n_groups=n_groups, + # condition_type=condition_type + # ), + ]) + + down_modules = nn.ModuleList([]) + for ind, (dim_in, dim_out) in enumerate(in_out): + is_last = ind >= (len(in_out) - 1) + down_modules.append( + nn.ModuleList([ + ConditionalResidualBlock1D( + dim_in, + dim_out, + cond_dim=cond_dim, + kernel_size=kernel_size, + n_groups=n_groups, + condition_type=condition_type, + ), + # ConditionalResidualBlock1D( + # dim_out, dim_out, cond_dim=cond_dim, + # kernel_size=kernel_size, n_groups=n_groups, + # condition_type=condition_type), + Downsample1d(dim_out) if not is_last else nn.Identity(), + ])) + + up_modules = nn.ModuleList([]) + for ind, (dim_in, dim_out) in enumerate(reversed(in_out[1:])): + is_last = ind >= (len(in_out) - 1) + up_modules.append( + nn.ModuleList([ + ConditionalResidualBlock1D( + dim_out * 2, + dim_in, + cond_dim=cond_dim, + kernel_size=kernel_size, + n_groups=n_groups, + condition_type=condition_type, + ), + # ConditionalResidualBlock1D( + # dim_in, dim_in, cond_dim=cond_dim, + # kernel_size=kernel_size, n_groups=n_groups, + # condition_type=condition_type), + Upsample1d(dim_in) if not is_last else nn.Identity(), + ])) + + final_conv = nn.Sequential( + Conv1dBlock(start_dim, start_dim, kernel_size=kernel_size), + nn.Conv1d(start_dim, input_dim, 1), + ) + + self.diffusion_step_encoder = diffusion_step_encoder + self.local_cond_encoder = local_cond_encoder + self.up_modules = up_modules + self.down_modules = down_modules + self.final_conv = final_conv + + logger.info("number of parameters: %e", sum(p.numel() for p in self.parameters())) + print_params(self) + + def forward( + self, + sample: torch.Tensor, + timestep: Union[torch.Tensor, float, int], + local_cond=None, + global_cond=None, + **kwargs, + ): + """ + x: (B,T,input_dim) + timestep: (B,) or int, diffusion step + local_cond: (B,T,local_cond_dim) + global_cond: (B,global_cond_dim) + output: (B,T,input_dim) + """ + sample = einops.rearrange(sample, "b h t -> b t h") + + # 1. time + timesteps = timestep + if not torch.is_tensor(timesteps): + # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can + timesteps = torch.tensor([timesteps], dtype=torch.long, device=sample.device) + elif torch.is_tensor(timesteps) and len(timesteps.shape) == 0: + timesteps = timesteps[None].to(sample.device) + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timesteps = timesteps.expand(sample.shape[0]) + + timestep_embed = self.diffusion_step_encoder(timesteps) + if global_cond is not None: + global_feature = torch.cat([timestep_embed, global_cond], axis=-1) + + # encode local features + h_local = list() + if local_cond is not None: + local_cond = einops.rearrange(local_cond, "b h t -> b t h") + resnet, resnet2 = self.local_cond_encoder + x = resnet(local_cond, global_feature) + h_local.append(x) + x = resnet2(local_cond, global_feature) + h_local.append(x) + + x = sample + h = [] + for idx, (resnet, downsample) in enumerate(self.down_modules): + if self.use_down_condition: + x = resnet(x, global_feature) + # print(f'down1 {idx}: {x.shape}') + if idx == 0 and len(h_local) > 0: + x = x + h_local[0] + # x = resnet2(x, global_feature) + # print(f'down2 {idx}: {x.shape}') + else: + x = resnet(x) + if idx == 0 and len(h_local) > 0: + x = x + h_local[0] + x = resnet2(x) + h.append(x) + x = downsample(x) + + for mid_module in self.mid_modules: + if self.use_mid_condition: + x = mid_module(x, global_feature) + # print(f'mid1: {x.shape}') + else: + x = mid_module(x) + + for idx, (resnet, upsample) in enumerate(self.up_modules): + x = torch.cat((x, h.pop()), dim=1) + if self.use_up_condition: + x = resnet(x, global_feature) + # print(f'up1 {idx}: {x.shape}') + if idx == len(self.up_modules) and len(h_local) > 0: + x = x + h_local[1] + # x = resnet2(x, global_feature) + # print(f'up2 {idx}: {x.shape}') + else: + x = resnet(x) + if idx == len(self.up_modules) and len(h_local) > 0: + x = x + h_local[1] + x = resnet2(x) + x = upsample(x) + + x = self.final_conv(x) + # print(f'final: {x.shape}') + + x = einops.rearrange(x, "b t h -> b h t") + return x diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/vision/pointnet_extractor.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/vision/pointnet_extractor.py new file mode 100644 index 00000000..1aa15e80 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/model/vision/pointnet_extractor.py @@ -0,0 +1,268 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision +import copy + +from typing import Optional, Dict, Tuple, Union, List, Type +from termcolor import cprint +import pdb + + +def create_mlp( + input_dim: int, + output_dim: int, + net_arch: List[int], + activation_fn: Type[nn.Module] = nn.ReLU, + squash_output: bool = False, +) -> List[nn.Module]: + """ + Create a multi layer perceptron (MLP), which is + a collection of fully-connected layers each followed by an activation function. + + :param input_dim: Dimension of the input vector + :param output_dim: + :param net_arch: Architecture of the neural net + It represents the number of units per layer. + The length of this list is the number of layers. + :param activation_fn: The activation function + to use after each layer. + :param squash_output: Whether to squash the output using a Tanh + activation function + :return: + """ + + if len(net_arch) > 0: + modules = [nn.Linear(input_dim, net_arch[0]), activation_fn()] + else: + modules = [] + + for idx in range(len(net_arch) - 1): + modules.append(nn.Linear(net_arch[idx], net_arch[idx + 1])) + modules.append(activation_fn()) + + if output_dim > 0: + last_layer_dim = net_arch[-1] if len(net_arch) > 0 else input_dim + modules.append(nn.Linear(last_layer_dim, output_dim)) + if squash_output: + modules.append(nn.Tanh()) + return modules + + +class PointNetEncoderXYZRGB(nn.Module): + """Encoder for Pointcloud""" + + def __init__( + self, + in_channels: int, + out_channels: int = 1024, + use_layernorm: bool = False, + final_norm: str = "none", + use_projection: bool = True, + **kwargs, + ): + """_summary_ + + Args: + in_channels (int): feature size of input (3 or 6) + input_transform (bool, optional): whether to use transformation for coordinates. Defaults to True. + feature_transform (bool, optional): whether to use transformation for features. Defaults to True. + is_seg (bool, optional): for segmentation or classification. Defaults to False. + """ + super().__init__() + block_channel = [64, 128, 256, 512] + cprint("pointnet use_layernorm: {}".format(use_layernorm), "cyan") + cprint("pointnet use_final_norm: {}".format(final_norm), "cyan") + + self.mlp = nn.Sequential( + nn.Linear(in_channels, block_channel[0]), + nn.LayerNorm(block_channel[0]) if use_layernorm else nn.Identity(), + nn.ReLU(), + nn.Linear(block_channel[0], block_channel[1]), + nn.LayerNorm(block_channel[1]) if use_layernorm else nn.Identity(), + nn.ReLU(), + nn.Linear(block_channel[1], block_channel[2]), + nn.LayerNorm(block_channel[2]) if use_layernorm else nn.Identity(), + nn.ReLU(), + nn.Linear(block_channel[2], block_channel[3]), + ) + + if final_norm == "layernorm": + self.final_projection = nn.Sequential(nn.Linear(block_channel[-1], out_channels), + nn.LayerNorm(out_channels)) + elif final_norm == "none": + self.final_projection = nn.Linear(block_channel[-1], out_channels) + else: + raise NotImplementedError(f"final_norm: {final_norm}") + + def forward(self, x): + x = self.mlp(x) + x = torch.max(x, 1)[0] + x = self.final_projection(x) + return x + + +class PointNetEncoderXYZ(nn.Module): + """Encoder for Pointcloud""" + + def __init__( + self, + in_channels: int = 3, + out_channels: int = 1024, + use_layernorm: bool = False, + final_norm: str = "none", + use_projection: bool = True, + **kwargs, + ): + """_summary_ + + Args: + in_channels (int): feature size of input (3 or 6) + input_transform (bool, optional): whether to use transformation for coordinates. Defaults to True. + feature_transform (bool, optional): whether to use transformation for features. Defaults to True. + is_seg (bool, optional): for segmentation or classification. Defaults to False. + """ + super().__init__() + block_channel = [64, 128, 256] + cprint("[PointNetEncoderXYZ] use_layernorm: {}".format(use_layernorm), "cyan") + cprint("[PointNetEncoderXYZ] use_final_norm: {}".format(final_norm), "cyan") + + assert in_channels == 3, cprint(f"PointNetEncoderXYZ only supports 3 channels, but got {in_channels}", "red") + + self.mlp = nn.Sequential( + nn.Linear(in_channels, block_channel[0]), + nn.LayerNorm(block_channel[0]) if use_layernorm else nn.Identity(), + nn.ReLU(), + nn.Linear(block_channel[0], block_channel[1]), + nn.LayerNorm(block_channel[1]) if use_layernorm else nn.Identity(), + nn.ReLU(), + nn.Linear(block_channel[1], block_channel[2]), + nn.LayerNorm(block_channel[2]) if use_layernorm else nn.Identity(), + nn.ReLU(), + ) + + if final_norm == "layernorm": + self.final_projection = nn.Sequential(nn.Linear(block_channel[-1], out_channels), + nn.LayerNorm(out_channels)) + elif final_norm == "none": + self.final_projection = nn.Linear(block_channel[-1], out_channels) + else: + raise NotImplementedError(f"final_norm: {final_norm}") + + self.use_projection = use_projection + if not use_projection: + self.final_projection = nn.Identity() + cprint("[PointNetEncoderXYZ] not use projection", "yellow") + + VIS_WITH_GRAD_CAM = False + if VIS_WITH_GRAD_CAM: + self.gradient = None + self.feature = None + self.input_pointcloud = None + self.mlp[0].register_forward_hook(self.save_input) + self.mlp[6].register_forward_hook(self.save_feature) + self.mlp[6].register_backward_hook(self.save_gradient) + + def forward(self, x): + x = self.mlp(x) + x = torch.max(x, 1)[0] + x = self.final_projection(x) + return x + + def save_gradient(self, module, grad_input, grad_output): + """ + for grad-cam + """ + self.gradient = grad_output[0] + + def save_feature(self, module, input, output): + """ + for grad-cam + """ + if isinstance(output, tuple): + self.feature = output[0].detach() + else: + self.feature = output.detach() + + def save_input(self, module, input, output): + """ + for grad-cam + """ + self.input_pointcloud = input[0].detach() + + +class DP3Encoder(nn.Module): + + def __init__( + self, + observation_space: Dict, + img_crop_shape=None, + out_channel=256, + state_mlp_size=(64, 64), + state_mlp_activation_fn=nn.ReLU, + pointcloud_encoder_cfg=None, + use_pc_color=False, + pointnet_type="pointnet", + ): + super().__init__() + self.imagination_key = "imagin_robot" + self.state_key = "agent_pos" + self.point_cloud_key = "point_cloud" + self.rgb_image_key = "image" + self.n_output_channels = out_channel + + self.use_imagined_robot = self.imagination_key in observation_space.keys() + self.point_cloud_shape = observation_space[self.point_cloud_key] + self.state_shape = observation_space[self.state_key] + if self.use_imagined_robot: + self.imagination_shape = observation_space[self.imagination_key] + else: + self.imagination_shape = None + + cprint(f"[DP3Encoder] point cloud shape: {self.point_cloud_shape}", "yellow") + cprint(f"[DP3Encoder] state shape: {self.state_shape}", "yellow") + cprint(f"[DP3Encoder] imagination point shape: {self.imagination_shape}", "yellow") + + self.use_pc_color = use_pc_color + self.pointnet_type = pointnet_type + if pointnet_type == "pointnet": + if use_pc_color: + pointcloud_encoder_cfg.in_channels = 6 + self.extractor = PointNetEncoderXYZRGB(**pointcloud_encoder_cfg) + else: + pointcloud_encoder_cfg.in_channels = 3 + self.extractor = PointNetEncoderXYZ(**pointcloud_encoder_cfg) + else: + raise NotImplementedError(f"pointnet_type: {pointnet_type}") + + if len(state_mlp_size) == 0: + raise RuntimeError(f"State mlp size is empty") + elif len(state_mlp_size) == 1: + net_arch = [] + else: + net_arch = state_mlp_size[:-1] + output_dim = state_mlp_size[-1] + + self.n_output_channels += output_dim + self.state_mlp = nn.Sequential(*create_mlp(self.state_shape[0], output_dim, net_arch, state_mlp_activation_fn)) + + cprint(f"[DP3Encoder] output dim: {self.n_output_channels}", "red") + + def forward(self, observations: Dict) -> torch.Tensor: + points = observations[self.point_cloud_key] + assert len(points.shape) == 3, cprint(f"point cloud shape: {points.shape}, length should be 3", "red") + if self.use_imagined_robot: + img_points = observations[self.imagination_key][..., :points.shape[-1]] # align the last dim + points = torch.concat([points, img_points], dim=1) + + # points = torch.transpose(points, 1, 2) # B * 3 * N + # points: B * 3 * (N + sum(Ni)) + pn_feat = self.extractor(points) # B * out_channel + + state = observations[self.state_key] + state_feat = self.state_mlp(state) # B * 64 + final_feat = torch.cat([pn_feat, state_feat], dim=-1) + return final_feat + + def output_shape(self): + return self.n_output_channels diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/policy/base_policy.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/policy/base_policy.py new file mode 100644 index 00000000..ac7aba0f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/policy/base_policy.py @@ -0,0 +1,26 @@ +from typing import Dict +import torch +import torch.nn as nn +from diffusion_policy_3d.model.common.module_attr_mixin import ModuleAttrMixin +from diffusion_policy_3d.model.common.normalizer import LinearNormalizer + + +class BasePolicy(ModuleAttrMixin): + # init accepts keyword argument shape_meta, see config/task/*_image.yaml + + def predict_action(self, obs_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """ + obs_dict: + str: B,To,* + return: B,Ta,Da + """ + raise NotImplementedError() + + # reset state for stateful policies + def reset(self): + pass + + # ========== training =========== + # no standard training interface except setting normalizer + def set_normalizer(self, normalizer: LinearNormalizer): + raise NotImplementedError() diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/policy/dp3.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/policy/dp3.py new file mode 100644 index 00000000..a0e80b1c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/diffusion_policy_3d/policy/dp3.py @@ -0,0 +1,382 @@ +from typing import Dict +import math +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange, reduce +from diffusers.schedulers.scheduling_ddpm import DDPMScheduler +from termcolor import cprint +import copy +import time +import pdb + +# import pytorch3d.ops as torch3d_ops + +from diffusion_policy_3d.model.common.normalizer import LinearNormalizer +from diffusion_policy_3d.policy.base_policy import BasePolicy +from diffusion_policy_3d.model.diffusion.conditional_unet1d import ConditionalUnet1D +from diffusion_policy_3d.model.diffusion.mask_generator import LowdimMaskGenerator +from diffusion_policy_3d.common.pytorch_util import dict_apply +from diffusion_policy_3d.common.model_util import print_params +from diffusion_policy_3d.model.vision.pointnet_extractor import DP3Encoder + + +class DP3(BasePolicy): + + def __init__( + self, + shape_meta: dict, + noise_scheduler: DDPMScheduler, + horizon, + n_action_steps, + n_obs_steps, + num_inference_steps=None, + obs_as_global_cond=True, + diffusion_step_embed_dim=256, + down_dims=(256, 512, 1024), + kernel_size=5, + n_groups=8, + condition_type="film", + use_down_condition=True, + use_mid_condition=True, + use_up_condition=True, + encoder_output_dim=256, + crop_shape=None, + use_pc_color=False, + pointnet_type="pointnet", + pointcloud_encoder_cfg=None, + # parameters passed to step + **kwargs, + ): + super().__init__() + + self.condition_type = condition_type + + # parse shape_meta + action_shape = shape_meta["action"]["shape"] + self.action_shape = action_shape + if len(action_shape) == 1: + action_dim = action_shape[0] + elif len(action_shape) == 2: # use multiple hands + action_dim = action_shape[0] * action_shape[1] + else: + raise NotImplementedError(f"Unsupported action shape {action_shape}") + + obs_shape_meta = shape_meta["obs"] + obs_dict = dict_apply(obs_shape_meta, lambda x: x["shape"]) + + obs_encoder = DP3Encoder( + observation_space=obs_dict, + img_crop_shape=crop_shape, + out_channel=encoder_output_dim, + pointcloud_encoder_cfg=pointcloud_encoder_cfg, + use_pc_color=use_pc_color, + pointnet_type=pointnet_type, + ) + + # create diffusion model + obs_feature_dim = obs_encoder.output_shape() + input_dim = action_dim + obs_feature_dim + global_cond_dim = None + if obs_as_global_cond: + input_dim = action_dim + if "cross_attention" in self.condition_type: + global_cond_dim = obs_feature_dim + else: + global_cond_dim = obs_feature_dim * n_obs_steps + + self.use_pc_color = use_pc_color + self.pointnet_type = pointnet_type + cprint( + f"[DiffusionUnetHybridPointcloudPolicy] use_pc_color: {self.use_pc_color}", + "yellow", + ) + cprint( + f"[DiffusionUnetHybridPointcloudPolicy] pointnet_type: {self.pointnet_type}", + "yellow", + ) + + model = ConditionalUnet1D( + input_dim=input_dim, + local_cond_dim=None, + global_cond_dim=global_cond_dim, + diffusion_step_embed_dim=diffusion_step_embed_dim, + down_dims=down_dims, + kernel_size=kernel_size, + n_groups=n_groups, + condition_type=condition_type, + use_down_condition=use_down_condition, + use_mid_condition=use_mid_condition, + use_up_condition=use_up_condition, + ) + + self.obs_encoder = obs_encoder + self.model = model + self.noise_scheduler = noise_scheduler + + self.noise_scheduler_pc = copy.deepcopy(noise_scheduler) + self.mask_generator = LowdimMaskGenerator( + action_dim=action_dim, + obs_dim=0 if obs_as_global_cond else obs_feature_dim, + max_n_obs_steps=n_obs_steps, + fix_obs_steps=True, + action_visible=False, + ) + + self.normalizer = LinearNormalizer() + self.horizon = horizon + self.obs_feature_dim = obs_feature_dim + self.action_dim = action_dim + self.n_action_steps = n_action_steps + self.n_obs_steps = n_obs_steps + self.obs_as_global_cond = obs_as_global_cond + self.kwargs = kwargs + + if num_inference_steps is None: + num_inference_steps = noise_scheduler.config.num_train_timesteps + self.num_inference_steps = num_inference_steps + + print_params(self) + + # ========= inference ============ + def conditional_sample( + self, + condition_data, + condition_mask, + condition_data_pc=None, + condition_mask_pc=None, + local_cond=None, + global_cond=None, + generator=None, + # keyword arguments to scheduler.step + **kwargs, + ): + model = self.model + scheduler = self.noise_scheduler + + trajectory = torch.randn( + size=condition_data.shape, + dtype=condition_data.dtype, + device=condition_data.device, + ) + + # set step values + scheduler.set_timesteps(self.num_inference_steps) + + for t in scheduler.timesteps: + # 1. apply conditioning + trajectory[condition_mask] = condition_data[condition_mask] + + model_output = model( + sample=trajectory, + timestep=t, + local_cond=local_cond, + global_cond=global_cond, + ) + + # 3. compute previous image: x_t -> x_t-1 + trajectory = scheduler.step( + model_output, + t, + trajectory, + ).prev_sample + + # finally make sure conditioning is enforced + trajectory[condition_mask] = condition_data[condition_mask] + + return trajectory + + def predict_action(self, obs_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """ + obs_dict: must include "obs" key + result: must include "action" key + """ + # normalize input + nobs = self.normalizer.normalize(obs_dict) + # this_n_point_cloud = nobs['imagin_robot'][..., :3] # only use coordinate + if not self.use_pc_color: + nobs["point_cloud"] = nobs["point_cloud"][..., :3] + this_n_point_cloud = nobs["point_cloud"] + + value = next(iter(nobs.values())) + B, To = value.shape[:2] + T = self.horizon + Da = self.action_dim + Do = self.obs_feature_dim + To = self.n_obs_steps + + # build input + device = self.device + dtype = self.dtype + + # handle different ways of passing observation + local_cond = None + global_cond = None + if self.obs_as_global_cond: + # condition through global feature + this_nobs = dict_apply(nobs, lambda x: x[:, :To, ...].reshape(-1, *x.shape[2:])) + nobs_features = self.obs_encoder(this_nobs) + if "cross_attention" in self.condition_type: + # treat as a sequence + global_cond = nobs_features.reshape(B, self.n_obs_steps, -1) + else: + # reshape back to B, Do + global_cond = nobs_features.reshape(B, -1) + # empty data for action + cond_data = torch.zeros(size=(B, T, Da), device=device, dtype=dtype) + cond_mask = torch.zeros_like(cond_data, dtype=torch.bool) + else: + # condition through impainting + this_nobs = dict_apply(nobs, lambda x: x[:, :To, ...].reshape(-1, *x.shape[2:])) + nobs_features = self.obs_encoder(this_nobs) + # reshape back to B, T, Do + nobs_features = nobs_features.reshape(B, To, -1) + cond_data = torch.zeros(size=(B, T, Da + Do), device=device, dtype=dtype) + cond_mask = torch.zeros_like(cond_data, dtype=torch.bool) + cond_data[:, :To, Da:] = nobs_features + cond_mask[:, :To, Da:] = True + + # run sampling + nsample = self.conditional_sample( + cond_data, + cond_mask, + local_cond=local_cond, + global_cond=global_cond, + **self.kwargs, + ) + + # unnormalize prediction + naction_pred = nsample[..., :Da] + action_pred = self.normalizer["action"].unnormalize(naction_pred) + + # get action + start = To - 1 + end = start + self.n_action_steps + action = action_pred[:, start:end] + + # get prediction + result = { + "action": action, + "action_pred": action_pred, + } + + return result + + # ========= training ============ + def set_normalizer(self, normalizer: LinearNormalizer): + self.normalizer.load_state_dict(normalizer.state_dict()) + + def compute_loss(self, batch): + # normalize input + + nobs = self.normalizer.normalize(batch["obs"]) + nactions = self.normalizer["action"].normalize(batch["action"]) + + if not self.use_pc_color: + nobs["point_cloud"] = nobs["point_cloud"][..., :3] + + batch_size = nactions.shape[0] + horizon = nactions.shape[1] + + # handle different ways of passing observation + local_cond = None + global_cond = None + trajectory = nactions + cond_data = trajectory + + if self.obs_as_global_cond: + # reshape B, T, ... to B*T + this_nobs = dict_apply(nobs, lambda x: x[:, :self.n_obs_steps, ...].reshape(-1, *x.shape[2:])) + nobs_features = self.obs_encoder(this_nobs) + + if "cross_attention" in self.condition_type: + # treat as a sequence + global_cond = nobs_features.reshape(batch_size, self.n_obs_steps, -1) + else: + # reshape back to B, Do + global_cond = nobs_features.reshape(batch_size, -1) + # this_n_point_cloud = this_nobs['imagin_robot'].reshape(batch_size,-1, *this_nobs['imagin_robot'].shape[1:]) + this_n_point_cloud = this_nobs["point_cloud"].reshape(batch_size, -1, *this_nobs["point_cloud"].shape[1:]) + this_n_point_cloud = this_n_point_cloud[..., :3] + else: + # reshape B, T, ... to B*T + this_nobs = dict_apply(nobs, lambda x: x.reshape(-1, *x.shape[2:])) + nobs_features = self.obs_encoder(this_nobs) + # reshape back to B, T, Do + nobs_features = nobs_features.reshape(batch_size, horizon, -1) + cond_data = torch.cat([nactions, nobs_features], dim=-1) + trajectory = cond_data.detach() + + # generate impainting mask + condition_mask = self.mask_generator(trajectory.shape) + + # Sample noise that we'll add to the images + noise = torch.randn(trajectory.shape, device=trajectory.device) + + bsz = trajectory.shape[0] + # Sample a random timestep for each image + timesteps = torch.randint( + 0, + self.noise_scheduler.config.num_train_timesteps, + (bsz, ), + device=trajectory.device, + ).long() + + # Add noise to the clean images according to the noise magnitude at each timestep + # (this is the forward diffusion process) + noisy_trajectory = self.noise_scheduler.add_noise(trajectory, noise, timesteps) + + # compute loss mask + loss_mask = ~condition_mask + + # apply conditioning + noisy_trajectory[condition_mask] = cond_data[condition_mask] + + # Predict the noise residual + + pred = self.model( + sample=noisy_trajectory, + timestep=timesteps, + local_cond=local_cond, + global_cond=global_cond, + ) + + pred_type = self.noise_scheduler.config.prediction_type + if pred_type == "epsilon": + target = noise + elif pred_type == "sample": + target = trajectory + elif pred_type == "v_prediction": + # https://github.com/huggingface/diffusers/blob/main/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py + # https://github.com/huggingface/diffusers/blob/v0.11.1-patch/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py + # sigma = self.noise_scheduler.sigmas[timesteps] + # alpha_t, sigma_t = self.noise_scheduler._sigma_to_alpha_sigma_t(sigma) + self.noise_scheduler.alpha_t = self.noise_scheduler.alpha_t.to(self.device) + self.noise_scheduler.sigma_t = self.noise_scheduler.sigma_t.to(self.device) + alpha_t, sigma_t = ( + self.noise_scheduler.alpha_t[timesteps], + self.noise_scheduler.sigma_t[timesteps], + ) + alpha_t = alpha_t.unsqueeze(-1).unsqueeze(-1) + sigma_t = sigma_t.unsqueeze(-1).unsqueeze(-1) + v_t = alpha_t * noise - sigma_t * trajectory + target = v_t + else: + raise ValueError(f"Unsupported prediction type {pred_type}") + + loss = F.mse_loss(pred, target, reduction="none") + loss = loss * loss_mask.type(loss.dtype) + loss = reduce(loss, "b ... -> b (...)", "mean") + loss = loss.mean() + + loss_dict = { + "bc_loss": loss.item(), + } + + # print(f"t2-t1: {t2-t1:.3f}") + # print(f"t3-t2: {t3-t2:.3f}") + # print(f"t4-t3: {t4-t3:.3f}") + # print(f"t5-t4: {t5-t4:.3f}") + # print(f"t6-t5: {t6-t5:.3f}") + + return loss, loss_dict diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/dp3_policy.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/dp3_policy.py new file mode 100644 index 00000000..f7f8bcf6 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/dp3_policy.py @@ -0,0 +1,48 @@ +if __name__ == "__main__": + import sys + import os + import pathlib + + ROOT_DIR = str(pathlib.Path(__file__).parent.parent.parent) + sys.path.append(ROOT_DIR) + os.chdir(ROOT_DIR) + +import os +import hydra +from omegaconf import OmegaConf +import pathlib +import sys +from train_dp3 import TrainDP3Workspace + +OmegaConf.register_new_resolver("eval", eval, replace=True) + + +@hydra.main( + version_base=None, + config_path=str(pathlib.Path(__file__).parent.joinpath("diffusion_policy_3d", "config")), +) +def main(cfg): + workspace = TrainDP3Workspace(cfg) + workspace.eval() + + +class DP3: + + def __init__(self, cfg, usr_args) -> None: + self.policy, self.env_runner = self.get_policy_and_runner(cfg, usr_args) + + def update_obs(self, observation): + self.env_runner.update_obs(observation) + + def get_action(self, observation=None): + action = self.env_runner.get_action(self.policy, observation) + return action + + def get_policy_and_runner(self, cfg, usr_args): + workspace = TrainDP3Workspace(cfg) + policy, env_runner = workspace.get_policy_and_runner(cfg, usr_args) + return policy, env_runner + + +if __name__ == "__main__": + main() diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/setup.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/setup.py new file mode 100644 index 00000000..a54f6d7f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/setup.py @@ -0,0 +1,6 @@ +from setuptools import setup, find_packages + +setup( + name="diffusion_policy_3d", + packages=find_packages(), +) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/train_dp3.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/train_dp3.py new file mode 100644 index 00000000..17860de9 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/3D-Diffusion-Policy/train_dp3.py @@ -0,0 +1,473 @@ +if __name__ == "__main__": + import sys + import os + import pathlib + + ROOT_DIR = str(pathlib.Path(__file__).parent.parent) + sys.path.append(ROOT_DIR) + os.chdir(ROOT_DIR) + +import os, sys +import pdb +import hydra +import torch +import dill +from omegaconf import OmegaConf +import pathlib + +DP3_ROOT = str(pathlib.Path(__file__).parent.parent) + +sys.path.append(DP3_ROOT) +sys.path.append(os.path.join(DP3_ROOT, '3D-Diffusion-Policy')) +sys.path.append(os.path.join(DP3_ROOT, '3D-Diffusion-Policy', 'diffusion_policy_3d')) + +from torch.utils.data import DataLoader +import copy + +import wandb +import tqdm +import numpy as np +from termcolor import cprint +import shutil +import time +import threading +import sys + +from hydra.core.hydra_config import HydraConfig +from diffusion_policy_3d.policy.dp3 import DP3 +from diffusion_policy_3d.dataset.base_dataset import BaseDataset +from diffusion_policy_3d.env_runner.base_runner import BaseRunner +from diffusion_policy_3d.env_runner.robot_runner import RobotRunner +from diffusion_policy_3d.common.checkpoint_util import TopKCheckpointManager +from diffusion_policy_3d.common.pytorch_util import dict_apply, optimizer_to +from diffusion_policy_3d.model.diffusion.ema_model import EMAModel +from diffusion_policy_3d.model.common.lr_scheduler import get_scheduler + +import pdb, random + +OmegaConf.register_new_resolver("eval", eval, replace=True) + + +class TrainDP3Workspace: + include_keys = ["global_step", "epoch"] + exclude_keys = tuple() + + def __init__(self, cfg: OmegaConf, output_dir=None): + self.cfg = cfg + self._output_dir = output_dir + self._saving_thread = None + + # set seed + seed = cfg.training.seed + torch.manual_seed(seed) + np.random.seed(seed) + random.seed(seed) + + # configure model + self.model: DP3 = hydra.utils.instantiate(cfg.policy) + + self.ema_model: DP3 = None + if cfg.training.use_ema: + try: + self.ema_model = copy.deepcopy(self.model) + except: # minkowski engine could not be copied. recreate it + self.ema_model = hydra.utils.instantiate(cfg.policy) + + # configure training state + self.optimizer = hydra.utils.instantiate(cfg.optimizer, params=self.model.parameters()) + + # configure training state + self.global_step = 0 + self.epoch = 0 + + def run(self): + cfg = copy.deepcopy(self.cfg) + + WANDB = False + + if cfg.training.debug: + cfg.training.num_epochs = 100 + cfg.training.max_train_steps = 10 + cfg.training.max_val_steps = 3 + cfg.training.rollout_every = 20 + cfg.training.checkpoint_every = 1 + cfg.training.val_every = 1 + cfg.training.sample_every = 1 + RUN_ROLLOUT = True + RUN_CKPT = False + verbose = True + else: + RUN_ROLLOUT = True + RUN_CKPT = True + verbose = False + + RUN_ROLLOUT = False + RUN_VALIDATION = True # reduce time cost + + # resume training + if cfg.training.resume: + lastest_ckpt_path = self.get_checkpoint_path() + if lastest_ckpt_path.is_file(): + print(f"Resuming from checkpoint {lastest_ckpt_path}") + self.load_checkpoint(path=lastest_ckpt_path) + + # configure dataset + dataset: BaseDataset + dataset = hydra.utils.instantiate(cfg.task.dataset) + + assert isinstance(dataset, BaseDataset), print(f"dataset must be BaseDataset, got {type(dataset)}") + train_dataloader = DataLoader(dataset, **cfg.dataloader) + normalizer = dataset.get_normalizer() + + # configure validation dataset + val_dataset = dataset.get_validation_dataset() + val_dataloader = DataLoader(val_dataset, **cfg.val_dataloader) + + self.model.set_normalizer(normalizer) + if cfg.training.use_ema: + self.ema_model.set_normalizer(normalizer) + + # configure lr scheduler + lr_scheduler = get_scheduler( + cfg.training.lr_scheduler, + optimizer=self.optimizer, + num_warmup_steps=cfg.training.lr_warmup_steps, + num_training_steps=(len(train_dataloader) * cfg.training.num_epochs) // + cfg.training.gradient_accumulate_every, + # pytorch assumes stepping LRScheduler every epoch + # however huggingface diffusers steps it every batch + last_epoch=self.global_step - 1, + ) + + # configure ema + ema: EMAModel = None + if cfg.training.use_ema: + ema = hydra.utils.instantiate(cfg.ema, model=self.ema_model) + + env_runner = None + + cfg.logging.name = str(cfg.task.name) + cprint("-----------------------------", "yellow") + cprint(f"[WandB] group: {cfg.logging.group}", "yellow") + cprint(f"[WandB] name: {cfg.logging.name}", "yellow") + cprint("-----------------------------", "yellow") + # configure logging + if WANDB: + wandb_run = wandb.init( + dir=str(self.output_dir), + config=OmegaConf.to_container(cfg, resolve=True), + **cfg.logging, + ) + wandb.config.update({ + "output_dir": self.output_dir, + }) + + # configure checkpoint + topk_manager = TopKCheckpointManager(save_dir=os.path.join(self.output_dir, "checkpoints"), + **cfg.checkpoint.topk) + + # device transfer + device = torch.device(cfg.training.device) + self.model.to(device) + if self.ema_model is not None: + self.ema_model.to(device) + optimizer_to(self.optimizer, device) + + # save batch for sampling + train_sampling_batch = None + checkpoint_num = 1 + + # training loop + log_path = os.path.join(self.output_dir, "logs.json.txt") + for local_epoch_idx in range(cfg.training.num_epochs): + step_log = dict() + # ========= train for this epoch ========== + train_losses = list() + with tqdm.tqdm( + train_dataloader, + desc=f"Training epoch {self.epoch}", + leave=False, + mininterval=cfg.training.tqdm_interval_sec, + ) as tepoch: + for batch_idx, batch in enumerate(tepoch): + t1 = time.time() + # device transfer + batch = dict_apply(batch, lambda x: x.to(device, non_blocking=True)) + if train_sampling_batch is None: + train_sampling_batch = batch + + # compute loss + t1_1 = time.time() + raw_loss, loss_dict = self.model.compute_loss(batch) + loss = raw_loss / cfg.training.gradient_accumulate_every + loss.backward() + + t1_2 = time.time() + + # step optimizer + if self.global_step % cfg.training.gradient_accumulate_every == 0: + self.optimizer.step() + self.optimizer.zero_grad() + lr_scheduler.step() + t1_3 = time.time() + # update ema + if cfg.training.use_ema: + ema.step(self.model) + t1_4 = time.time() + # logging + raw_loss_cpu = raw_loss.item() + tepoch.set_postfix(loss=raw_loss_cpu, refresh=False) + train_losses.append(raw_loss_cpu) + step_log = { + "train_loss": raw_loss_cpu, + "global_step": self.global_step, + "epoch": self.epoch, + "lr": lr_scheduler.get_last_lr()[0], + } + t1_5 = time.time() + step_log.update(loss_dict) + t2 = time.time() + + if verbose: + print(f"total one step time: {t2-t1:.3f}") + print(f" compute loss time: {t1_2-t1_1:.3f}") + print(f" step optimizer time: {t1_3-t1_2:.3f}") + print(f" update ema time: {t1_4-t1_3:.3f}") + print(f" logging time: {t1_5-t1_4:.3f}") + + is_last_batch = batch_idx == (len(train_dataloader) - 1) + if not is_last_batch: + # log of last step is combined with validation and rollout + if WANDB: + wandb_run.log(step_log, step=self.global_step) + self.global_step += 1 + + if (cfg.training.max_train_steps is not None) and batch_idx >= (cfg.training.max_train_steps - 1): + break + + # at the end of each epoch + # replace train_loss with epoch average + train_loss = np.mean(train_losses) + step_log["train_loss"] = train_loss + + # ========= eval for this epoch ========== + policy = self.model + if cfg.training.use_ema: + policy = self.ema_model + policy.eval() + + # run validation + if (self.epoch % cfg.training.val_every) == 0 and RUN_VALIDATION: + with torch.no_grad(): + val_losses = list() + with tqdm.tqdm( + val_dataloader, + desc=f"Validation epoch {self.epoch}", + leave=False, + mininterval=cfg.training.tqdm_interval_sec, + ) as tepoch: + for batch_idx, batch in enumerate(tepoch): + batch = dict_apply(batch, lambda x: x.to(device, non_blocking=True)) + loss, loss_dict = self.model.compute_loss(batch) + val_losses.append(loss) + print(f"epoch {self.epoch}, eval loss: ", float(loss.cpu())) + if (cfg.training.max_val_steps + is not None) and batch_idx >= (cfg.training.max_val_steps - 1): + break + if len(val_losses) > 0: + val_loss = torch.mean(torch.tensor(val_losses)).item() + # log epoch average validation loss + step_log["val_loss"] = val_loss + + # checkpoint + if ((self.epoch + 1) % cfg.training.checkpoint_every) == 0 and cfg.checkpoint.save_ckpt: + + if not cfg.policy.use_pc_color: + if not os.path.exists(f"checkpoints/{self.cfg.task.name}_{cfg.training.seed}"): + os.makedirs(f"checkpoints/{self.cfg.task.name}_{cfg.training.seed}") + save_path = f"checkpoints/{self.cfg.task.name}_{cfg.training.seed}/{self.epoch + 1}.ckpt" + else: + if not os.path.exists(f"checkpoints/{self.cfg.task.name}_w_rgb_{cfg.training.seed}"): + os.makedirs(f"checkpoints/{self.cfg.task.name}_w_rgb_{cfg.training.seed}") + save_path = f"checkpoints/{self.cfg.task.name}_w_rgb_{cfg.training.seed}/{self.epoch + 1}.ckpt" + + self.save_checkpoint(save_path) + + # ========= eval end for this epoch ========== + policy.train() + + # end of epoch + # log of last step is combined with validation and rollout + if WANDB: + wandb_run.log(step_log, step=self.global_step) + self.global_step += 1 + self.epoch += 1 + del step_log + + def get_policy_and_runner(self, cfg, usr_args): + # load the latest checkpoint + + cfg = copy.deepcopy(self.cfg) + + n_obs_steps = cfg['n_obs_steps'] + n_action_steps = cfg['n_action_steps'] + + env_runner = RobotRunner(n_obs_steps=n_obs_steps, n_action_steps=n_action_steps) + + if not cfg.policy.use_pc_color: + ckpt_file = pathlib.Path( + os.path.join( + DP3_ROOT, + f"./checkpoints/{usr_args['task_name']}-{usr_args['ckpt_setting']}-{usr_args['expert_data_num']}_{usr_args['seed']}/{usr_args['checkpoint_num']}.ckpt" + )) + else: + ckpt_file = pathlib.Path( + os.path.join( + DP3_ROOT, + f"./checkpoints/{usr_args['task_name']}-{usr_args['ckpt_setting']}-{usr_args['expert_data_num']}_w_rgb_{usr_args['seed']}/{usr_args['checkpoint_num']}.ckpt" + )) + assert ckpt_file.is_file(), f"ckpt file doesn't exist, {ckpt_file}" + + if ckpt_file.is_file(): + cprint(f"Resuming from checkpoint {ckpt_file}", "magenta") + self.load_checkpoint(path=ckpt_file) + + policy = self.model + if cfg.training.use_ema: + policy = self.ema_model + policy.eval() + policy.cuda() + return policy, env_runner + + @property + def output_dir(self): + output_dir = self._output_dir + if output_dir is None: + output_dir = HydraConfig.get().runtime.output_dir + return output_dir + + def save_checkpoint( + self, + path=None, + tag="latest", + exclude_keys=None, + include_keys=None, + use_thread=False, + ): + print("saved in ", path) + if path is None: + path = pathlib.Path(self.output_dir).joinpath("checkpoints", f"{tag}.ckpt") + else: + path = pathlib.Path(path) + if exclude_keys is None: + exclude_keys = tuple(self.exclude_keys) + if include_keys is None: + include_keys = tuple(self.include_keys) + ("_output_dir", ) + + path.parent.mkdir(parents=False, exist_ok=True) + payload = {"cfg": self.cfg, "state_dicts": dict(), "pickles": dict()} + + for key, value in self.__dict__.items(): + if hasattr(value, "state_dict") and hasattr(value, "load_state_dict"): + # modules, optimizers and samplers etc + if key not in exclude_keys: + if use_thread: + payload["state_dicts"][key] = _copy_to_cpu(value.state_dict()) + else: + payload["state_dicts"][key] = value.state_dict() + elif key in include_keys: + payload["pickles"][key] = dill.dumps(value) + if use_thread: + self._saving_thread = threading.Thread( + target=lambda: torch.save(payload, path.open("wb"), pickle_module=dill)) + self._saving_thread.start() + else: + torch.save(payload, path.open("wb"), pickle_module=dill) + + del payload + torch.cuda.empty_cache() + return str(path.absolute()) + + def get_checkpoint_path(self, tag="latest"): + if tag == "latest": + return pathlib.Path(self.output_dir).joinpath("checkpoints", f"{tag}.ckpt") + elif tag == "best": + # the checkpoints are saved as format: epoch={}-test_mean_score={}.ckpt + # find the best checkpoint + checkpoint_dir = pathlib.Path(self.output_dir).joinpath("checkpoints") + all_checkpoints = os.listdir(checkpoint_dir) + best_ckpt = None + best_score = -1e10 + for ckpt in all_checkpoints: + if "latest" in ckpt: + continue + score = float(ckpt.split("test_mean_score=")[1].split(".ckpt")[0]) + if score > best_score: + best_ckpt = ckpt + best_score = score + return pathlib.Path(self.output_dir).joinpath("checkpoints", best_ckpt) + else: + raise NotImplementedError(f"tag {tag} not implemented") + + def load_payload(self, payload, exclude_keys=None, include_keys=None, **kwargs): + if exclude_keys is None: + exclude_keys = tuple() + if include_keys is None: + include_keys = payload["pickles"].keys() + + for key, value in payload["state_dicts"].items(): + if key not in exclude_keys: + self.__dict__[key].load_state_dict(value, **kwargs) + for key in include_keys: + if key in payload["pickles"]: + self.__dict__[key] = dill.loads(payload["pickles"][key]) + + def load_checkpoint(self, path=None, tag="latest", exclude_keys=None, include_keys=None, **kwargs): + if path is None: + path = self.get_checkpoint_path(tag=tag) + else: + path = pathlib.Path(path) + payload = torch.load(path.open("rb"), pickle_module=dill, map_location="cpu") + self.load_payload(payload, exclude_keys=exclude_keys, include_keys=include_keys) + return payload + + @classmethod + def create_from_checkpoint(cls, path, exclude_keys=None, include_keys=None, **kwargs): + payload = torch.load(open(path, "rb"), pickle_module=dill) + instance = cls(payload["cfg"]) + instance.load_payload( + payload=payload, + exclude_keys=exclude_keys, + include_keys=include_keys, + **kwargs, + ) + return instance + + def save_snapshot(self, tag="latest"): + """ + Quick loading and saving for reserach, saves full state of the workspace. + + However, loading a snapshot assumes the code stays exactly the same. + Use save_checkpoint for long-term storage. + """ + path = pathlib.Path(self.output_dir).joinpath("snapshots", f"{tag}.pkl") + path.parent.mkdir(parents=False, exist_ok=True) + torch.save(self, path.open("wb"), pickle_module=dill) + return str(path.absolute()) + + @classmethod + def create_from_snapshot(cls, path): + return torch.load(open(path, "rb"), pickle_module=dill) + + +@hydra.main( + version_base=None, + config_path=str(pathlib.Path(__file__).parent.joinpath("diffusion_policy_3d", "config")), +) +def main(cfg): + workspace = TrainDP3Workspace(cfg) + workspace.run() + + +if __name__ == "__main__": + main() diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/__init__.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/__init__.py new file mode 100644 index 00000000..d4b67709 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/__init__.py @@ -0,0 +1 @@ +from .deploy_policy import * diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/deploy_policy.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/deploy_policy.py new file mode 100644 index 00000000..43c7de9c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/deploy_policy.py @@ -0,0 +1,95 @@ +# import packages and module here +import sys + +import torch +import sapien.core as sapien +import traceback +import os +import numpy as np +from envs import * +from hydra import initialize, compose +from omegaconf import OmegaConf +from hydra.core.hydra_config import HydraConfig +from hydra import main as hydra_main +import pathlib +from omegaconf import OmegaConf + +import yaml +from datetime import datetime +import importlib + +from hydra import initialize, compose +from omegaconf import OmegaConf +from datetime import datetime + +current_file_path = os.path.abspath(__file__) +parent_directory = os.path.dirname(current_file_path) + +sys.path.append(os.path.join(parent_directory, '3D-Diffusion-Policy')) + +from dp3_policy import * + + +def encode_obs(observation): # Post-Process Observation + obs = dict() + obs['agent_pos'] = observation['joint_action']['vector'] + obs['point_cloud'] = observation['pointcloud'] + return obs + + +def get_model(usr_args): + config_path = "./3D-Diffusion-Policy/diffusion_policy_3d/config" + config_name = f"{usr_args['config_name']}.yaml" + + with initialize(config_path=config_path, version_base='1.2'): + cfg = compose(config_name=config_name) + + now = datetime.now() + run_dir = f"data/outputs/{now:%Y.%m.%d}/{now:%H.%M.%S}_{usr_args['config_name']}_{usr_args['task_name']}" + + hydra_runtime_cfg = { + "job": { + "override_dirname": usr_args['task_name'] + }, + "run": { + "dir": run_dir + }, + "sweep": { + "dir": run_dir, + "subdir": "0" + } + } + + OmegaConf.set_struct(cfg, False) + cfg.hydra = hydra_runtime_cfg + cfg.task_name = usr_args["task_name"] + cfg.expert_data_num = usr_args["expert_data_num"] + cfg.raw_task_name = usr_args["task_name"] + cfg.policy.use_pc_color = usr_args['use_rgb'] + OmegaConf.set_struct(cfg, True) + + DP3_Model = DP3(cfg, usr_args) + return DP3_Model + + +def eval(TASK_ENV, model, observation): + obs = encode_obs(observation) # Post-Process Observation + # instruction = TASK_ENV.get_instruction() + + if len( + model.env_runner.obs + ) == 0: # Force an update of the observation at the first frame to avoid an empty observation window, `obs_cache` here can be modified + model.update_obs(obs) + + actions = model.get_action() # Get Action according to observation chunk + + for action in actions: # Execute each step of the action + TASK_ENV.take_action(action) + observation = TASK_ENV.get_obs() + obs = encode_obs(observation) + model.update_obs(obs) # Update Observation, `update_obs` here can be modified + + +def reset_model( + model): # Clean the model cache at the beginning of every evaluation episode, such as the observation window + model.env_runner.reset_obs() diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/deploy_policy.yml b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/deploy_policy.yml new file mode 100644 index 00000000..7f9287d8 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/deploy_policy.yml @@ -0,0 +1,14 @@ +# Basic experiment configuration (keep unchanged) +policy_name: null +task_name: null +task_config: null +ckpt_setting: null +seed: null +instruction_type: unseen + +# Add Parameters You Need +config_name: robot_dp3 +checkpoint_num: 3000 +dp3_task: demo_task +expert_data_num: null +use_rgb: false \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/eval.sh b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/eval.sh new file mode 100644 index 00000000..6d69d1ca --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/eval.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +policy_name=DP3 +task_name=${1} +task_config=${2} +ckpt_setting=${3} +expert_data_num=${4} +seed=${5} # both policy and RoboTwin scene +gpu_id=${6} + +export CUDA_VISIBLE_DEVICES=${gpu_id} +export HYDRA_FULL_ERROR=1 +echo -e "\033[33mgpu id (to use): ${gpu_id}\033[0m" + +cd ../.. # move to root + +PYTHONWARNINGS=ignore::UserWarning \ +python script/eval_policy.py --config policy/$policy_name/deploy_policy.yml \ + --overrides \ + --task_name ${task_name} \ + --task_config ${task_config} \ + --ckpt_setting ${ckpt_setting} \ + --expert_data_num ${expert_data_num} \ + --seed ${seed} \ + --policy_name ${policy_name} diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/process_data.sh b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/process_data.sh new file mode 100644 index 00000000..ac9b86be --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/process_data.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +task_name=${1} +task_config=${2} +expert_data_num=${3} + +python scripts/process_data.py $task_name $task_config $expert_data_num \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/scripts/process_data.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/scripts/process_data.py new file mode 100644 index 00000000..077f267f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/scripts/process_data.py @@ -0,0 +1,152 @@ +import pickle, os +import numpy as np +import pdb +from copy import deepcopy +import zarr +import shutil +import argparse +import yaml +import cv2 +import h5py + + +def load_hdf5(dataset_path): + if not os.path.isfile(dataset_path): + print(f"Dataset does not exist at \n{dataset_path}\n") + exit() + + with h5py.File(dataset_path, "r") as root: + left_gripper, left_arm = ( + root["/joint_action/left_gripper"][()], + root["/joint_action/left_arm"][()], + ) + right_gripper, right_arm = ( + root["/joint_action/right_gripper"][()], + root["/joint_action/right_arm"][()], + ) + vector = root["/joint_action/vector"][()] + pointcloud = root["/pointcloud"][()] + + return left_gripper, left_arm, right_gripper, right_arm, vector, pointcloud + + +def main(): + parser = argparse.ArgumentParser(description="Process some episodes.") + parser.add_argument( + "task_name", + type=str, + help="The name of the task (e.g., beat_block_hammer)", + ) + parser.add_argument("task_config", type=str) + parser.add_argument( + "expert_data_num", + type=int, + help="Number of episodes to process (e.g., 50)", + ) + args = parser.parse_args() + + task_name = args.task_name + num = args.expert_data_num + task_config = args.task_config + + load_dir = "../../data/" + str(task_name) + "/" + str(task_config) + + total_count = 0 + + save_dir = f"./data/{task_name}-{task_config}-{num}.zarr" + + if os.path.exists(save_dir): + shutil.rmtree(save_dir) + + current_ep = 0 + + zarr_root = zarr.group(save_dir) + zarr_data = zarr_root.create_group("data") + zarr_meta = zarr_root.create_group("meta") + + point_cloud_arrays = [] + episode_ends_arrays, action_arrays, state_arrays, joint_action_arrays = ( + [], + [], + [], + [], + ) + + while current_ep < num: + print(f"processing episode: {current_ep + 1} / {num}", end="\r") + + load_path = os.path.join(load_dir, f"data/episode{current_ep}.hdf5") + ( + left_gripper_all, + left_arm_all, + right_gripper_all, + right_arm_all, + vector_all, + pointcloud_all, + ) = load_hdf5(load_path) + + for j in range(0, left_gripper_all.shape[0]): + + pointcloud = pointcloud_all[j] + joint_state = vector_all[j] + + if j != left_gripper_all.shape[0] - 1: + point_cloud_arrays.append(pointcloud) + state_arrays.append(joint_state) + if j != 0: + joint_action_arrays.append(joint_state) + + current_ep += 1 + total_count += left_gripper_all.shape[0] - 1 + episode_ends_arrays.append(total_count) + + print() + try: + episode_ends_arrays = np.array(episode_ends_arrays) + state_arrays = np.array(state_arrays) + point_cloud_arrays = np.array(point_cloud_arrays) + joint_action_arrays = np.array(joint_action_arrays) + + compressor = zarr.Blosc(cname="zstd", clevel=3, shuffle=1) + state_chunk_size = (100, state_arrays.shape[1]) + joint_chunk_size = (100, joint_action_arrays.shape[1]) + point_cloud_chunk_size = (100, point_cloud_arrays.shape[1]) + zarr_data.create_dataset( + "point_cloud", + data=point_cloud_arrays, + chunks=point_cloud_chunk_size, + overwrite=True, + compressor=compressor, + ) + zarr_data.create_dataset( + "state", + data=state_arrays, + chunks=state_chunk_size, + dtype="float32", + overwrite=True, + compressor=compressor, + ) + zarr_data.create_dataset( + "action", + data=joint_action_arrays, + chunks=joint_chunk_size, + dtype="float32", + overwrite=True, + compressor=compressor, + ) + zarr_meta.create_dataset( + "episode_ends", + data=episode_ends_arrays, + dtype="int64", + overwrite=True, + compressor=compressor, + ) + except ZeroDivisionError as e: + print("If you get a `ZeroDivisionError: division by zero`, check that `data/pointcloud` in the task config is set to true.") + raise + except Exception as e: + print(f"An unexpected error occurred ({type(e).__name__}): {e}") + raise + +if __name__ == "__main__": + main() diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/scripts/train_policy.sh b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/scripts/train_policy.sh new file mode 100644 index 00000000..0bf3f7f5 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/scripts/train_policy.sh @@ -0,0 +1,47 @@ +DEBUG=False +save_ckpt=True + +alg_name=${1} +# task choices: See TASK.md +task_name=${2} +setting=${3} +expert_data_num=${4} +config_name=${alg_name} +addition_info=${5} +seed=${6} +exp_name=${task_name}-${alg_name}-${addition_info} +run_dir="data/outputs/${exp_name}_seed${seed}" + + +# gpu_id=$(bash scripts/find_gpu.sh) +gpu_id=${7} +echo -e "\033[33mgpu id (to use): ${gpu_id}\033[0m" + + +if [ $DEBUG = True ]; then + wandb_mode=offline + # wandb_mode=online + echo -e "\033[33mDebug mode!\033[0m" + echo -e "\033[33mDebug mode!\033[0m" + echo -e "\033[33mDebug mode!\033[0m" +else + wandb_mode=online + echo -e "\033[33mTrain mode\033[0m" +fi + +cd 3D-Diffusion-Policy + + +export HYDRA_FULL_ERROR=1 +export CUDA_VISIBLE_DEVICES=${gpu_id} +python train_dp3.py --config-name=${config_name}.yaml \ + task_name=${task_name} \ + hydra.run.dir=${run_dir} \ + training.debug=$DEBUG \ + training.seed=${seed} \ + training.device="cuda:0" \ + exp_name=${exp_name} \ + logging.mode=${wandb_mode} \ + checkpoint.save_ckpt=${save_ckpt} \ + expert_data_num=${expert_data_num} \ + setting=${setting} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/scripts/train_policy_rgb.sh b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/scripts/train_policy_rgb.sh new file mode 100644 index 00000000..afa5f65a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/scripts/train_policy_rgb.sh @@ -0,0 +1,48 @@ +DEBUG=False +save_ckpt=True + +alg_name=${1} +# task choices: See TASK.md +task_name=${2} +setting=${3} +expert_data_num=${4} +config_name=${alg_name} +addition_info=${5} +seed=${6} +exp_name=${task_name}-${alg_name}-${addition_info} +run_dir="data/outputs/${exp_name}_seed${seed}" + + +# gpu_id=$(bash scripts/find_gpu.sh) +gpu_id=${7} +echo -e "\033[33mgpu id (to use): ${gpu_id}\033[0m" + + +if [ $DEBUG = True ]; then + wandb_mode=offline + # wandb_mode=online + echo -e "\033[33mDebug mode!\033[0m" + echo -e "\033[33mDebug mode!\033[0m" + echo -e "\033[33mDebug mode!\033[0m" +else + wandb_mode=online + echo -e "\033[33mTrain mode\033[0m" +fi + +cd 3D-Diffusion-Policy + + +export HYDRA_FULL_ERROR=1 +export CUDA_VISIBLE_DEVICES=${gpu_id} +python train_dp3.py --config-name=${config_name}.yaml \ + task_name=${task_name} \ + hydra.run.dir=${run_dir} \ + training.debug=$DEBUG \ + training.seed=${seed} \ + training.device="cuda:0" \ + exp_name=${exp_name} \ + logging.mode=${wandb_mode} \ + checkpoint.save_ckpt=${save_ckpt} \ + expert_data_num=${expert_data_num} \ + setting=${setting} \ + policy.use_pc_color=True diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/train.sh b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/train.sh new file mode 100644 index 00000000..f675c23b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/train.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +task_name=${1} +task_config=${2} +expert_data_num=${3} +seed=${4} +gpu_id=${5} + +if [ ! -d "./data/${task_name}-${task_config}-${expert_data_num}.zarr" ]; then + bash process_data.sh ${task_name} ${task_config} ${expert_data_num} +fi + +bash scripts/train_policy.sh robot_dp3 ${task_name} ${task_config} ${expert_data_num} train ${seed} ${gpu_id} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/train_rgb.sh b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/train_rgb.sh new file mode 100644 index 00000000..4bbbcf58 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DP3/train_rgb.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +task_name=${1} +task_config=${2} +expert_data_num=${3} +seed=${4} +gpu_id=${5} + +if [ ! -d "./data/${task_name}-${task_config}-${expert_data_num}.zarr" ]; then + bash process_data.sh ${task_name} ${task_config} ${expert_data_num} +fi + +bash scripts/train_policy_rgb.sh robot_dp3 ${task_name} ${task_config} ${expert_data_num} train ${seed} ${gpu_id} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/Eval_Tiny_DexVLA_environment.yml b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/Eval_Tiny_DexVLA_environment.yml new file mode 100644 index 00000000..3a5bd78f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/Eval_Tiny_DexVLA_environment.yml @@ -0,0 +1,378 @@ +name: RoboTwin +channels: + - pytorch + - nvidia + - https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main + - conda-forge + - https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/pytorch/linux-64/ + - https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/pytorch/ + - https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main/ + - https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free/ + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - aom=3.6.1=h59595ed_0 + - blas=1.0=mkl + - brotlicffi=1.0.9.2=py310h6a678d5_1 + - bzip2=1.0.8=h5eee18b_6 + - ca-certificates=2025.6.15=hbd8a1cb_0 + - certifi=2025.6.15=pyhd8ed1ab_0 + - cffi=1.17.1=py310h1fdaa30_1 + - cuda-cudart=11.8.89=0 + - cuda-cupti=11.8.87=0 + - cuda-libraries=11.8.0=0 + - cuda-nvrtc=11.8.89=0 + - cuda-nvtx=11.8.86=0 + - cuda-runtime=11.8.0=0 + - cuda-version=12.9=3 + - expat=2.7.1=h6a678d5_0 + - ffmpeg=4.4.2=gpl_hdf48244_113 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_3 + - fontconfig=2.14.2=h14ed4e7_0 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - freetype=2.13.3=h4a9f257_0 + - gmp=6.3.0=h6a678d5_0 + - gmpy2=2.2.1=py310h5eee18b_0 + - gnutls=3.7.9=hb077bed_0 + - icu=73.2=h59595ed_0 + - jinja2=3.1.6=py310h06a4308_0 + - jpeg=9e=h5eee18b_3 + - lame=3.100=h7b6447c_0 + - lcms2=2.16=h92b89f2_1 + - ld_impl_linux-64=2.40=h12ee557_0 + - lerc=4.0.0=h6a678d5_0 + - libcublas=11.11.3.6=0 + - libcufft=10.9.0.58=0 + - libcufile=1.14.1.1=4 + - libcurand=10.3.10.19=0 + - libcusolver=11.4.1.48=0 + - libcusparse=11.7.5.86=0 + - libdeflate=1.22=h5eee18b_0 + - libdrm=2.4.125=hb9d3cd8_0 + - libffi=3.4.4=h6a678d5_1 + - libgcc=15.1.0=h767d61c_3 + - libgcc-ng=15.1.0=h69a702a_3 + - libgomp=15.1.0=h767d61c_3 + - libiconv=1.16=h5eee18b_3 + - libidn2=2.3.4=h5eee18b_0 + - libjpeg-turbo=2.0.0=h9bf148f_0 + - libnpp=11.8.0.86=0 + - libnsl=2.0.1=hb9d3cd8_1 + - libnvjpeg=11.9.0.86=0 + - libpciaccess=0.18=hb9d3cd8_0 + - libpng=1.6.39=h5eee18b_0 + - libsqlite=3.46.0=hde9e2c9_0 + - libstdcxx=15.1.0=h8f9b012_3 + - libstdcxx-ng=15.1.0=h4852527_3 + - libtasn1=4.19.0=h5eee18b_0 + - libtiff=4.7.0=hde9077f_0 + - libunistring=0.9.10=h27cfd23_0 + - libuuid=2.38.1=h0b41bf4_0 + - libva=2.21.0=h4ab18f5_1 + - libvpx=1.13.1=h59595ed_0 + - libwebp-base=1.3.2=h5eee18b_1 + - libxcb=1.17.0=h9b100fa_0 + - libxcrypt=4.4.36=hd590300_1 + - libxml2=2.13.8=hfdd30dd_0 + - libzlib=1.2.13=h4ab18f5_6 + - llvm-openmp=14.0.6=h9e868ea_0 + - lz4-c=1.9.4=h6a678d5_1 + - markupsafe=3.0.2=py310h5eee18b_0 + - mkl-service=2.4.0=py310h5eee18b_3 + - mkl_fft=1.3.11=py310hacdc0fc_1 + - mkl_random=1.2.8=py310h2fd27a0_1 + - mpc=1.3.1=h5eee18b_0 + - mpfr=4.2.1=h5eee18b_0 + - mpmath=1.3.0=py310h06a4308_0 + - ncurses=6.4=h6a678d5_0 + - nettle=3.9.1=h7ab15ed_0 + - openh264=2.3.1=hcb278e6_2 + - openjpeg=2.5.2=h0d4d230_1 + - openssl=3.5.1=h7b32b05_0 + - p11-kit=0.24.1=hc5aa10d_0 + - pip=25.1=pyhc872135_2 + - pthread-stubs=0.3=h0ce48e5_1 + - pysocks=1.7.1=py310h06a4308_0 + - python=3.10.13=hd12c33a_1_cpython + - pytorch=2.4.1=py3.10_cuda11.8_cudnn9.1.0_0 + - pytorch-cuda=11.8=h7e8668a_6 + - pytorch-mutex=1.0=cuda + - pyyaml=6.0.2=py310h5eee18b_0 + - readline=8.2=h5eee18b_0 + - requests=2.32.4=py310h06a4308_0 + - setuptools=78.1.1=py310h06a4308_0 + - sqlite=3.45.3=h5eee18b_0 + - svt-av1=1.4.1=hcb278e6_0 + - tbb-devel=2022.0.0=hdb19cb5_0 + - tk=8.6.14=h993c535_1 + - torchaudio=2.4.1=py310_cu118 + - torchtriton=3.0.0=py310 + - torchvision=0.19.1=py310_cu118 + - typing_extensions=4.12.2=py310h06a4308_0 + - urllib3=2.5.0=py310h06a4308_0 + - wheel=0.45.1=py310h06a4308_0 + - x264=1!164.3095=h166bdaf_2 + - x265=3.5=h924138e_3 + - xorg-libx11=1.8.12=h9b100fa_1 + - xorg-libxau=1.0.12=h9b100fa_0 + - xorg-libxdmcp=1.1.5=h9b100fa_0 + - xorg-libxext=1.3.6=hb9d3cd8_0 + - xorg-libxfixes=6.0.1=hb9d3cd8_0 + - xorg-xorgproto=2024.1=h5eee18b_1 + - xz=5.6.4=h5eee18b_1 + - yaml=0.2.5=h7b6447c_0 + - zlib=1.2.13=h4ab18f5_6 + - zstd=1.5.6=hc292b87_0 + - pip: + - accelerate==0.26.0 + - adal==1.2.7 + - addict==2.4.0 + - annotated-types==0.7.0 + - anyio==4.9.0 + - asciitree==0.3.3 + - asttokens==3.0.0 + - attrs==25.3.0 + - av==14.4.0 + - azure==4.0.0 + - azure-ai-inference==1.0.0b9 + - azure-applicationinsights==0.1.1 + - azure-batch==4.1.3 + - azure-common==1.1.28 + - azure-core==1.34.0 + - azure-cosmosdb-nspkg==2.0.2 + - azure-cosmosdb-table==1.0.6 + - azure-datalake-store==0.0.53 + - azure-eventgrid==1.3.0 + - azure-graphrbac==0.40.0 + - azure-keyvault==1.1.0 + - azure-loganalytics==0.1.1 + - azure-mgmt==4.0.0 + - azure-mgmt-advisor==1.0.1 + - azure-mgmt-applicationinsights==0.1.1 + - azure-mgmt-authorization==0.50.0 + - azure-mgmt-batch==5.0.1 + - azure-mgmt-batchai==2.0.0 + - azure-mgmt-billing==0.2.0 + - azure-mgmt-cdn==3.1.0 + - azure-mgmt-cognitiveservices==3.0.0 + - azure-mgmt-commerce==1.0.1 + - azure-mgmt-compute==4.6.2 + - azure-mgmt-consumption==2.0.0 + - azure-mgmt-containerinstance==1.5.0 + - azure-mgmt-containerregistry==2.8.0 + - azure-mgmt-containerservice==4.4.0 + - azure-mgmt-cosmosdb==0.4.1 + - azure-mgmt-datafactory==0.6.0 + - azure-mgmt-datalake-analytics==0.6.0 + - azure-mgmt-datalake-nspkg==3.0.1 + - azure-mgmt-datalake-store==0.5.0 + - azure-mgmt-datamigration==1.0.0 + - azure-mgmt-devspaces==0.1.0 + - azure-mgmt-devtestlabs==2.2.0 + - azure-mgmt-dns==2.1.0 + - azure-mgmt-eventgrid==1.0.0 + - azure-mgmt-eventhub==2.6.0 + - azure-mgmt-hanaonazure==0.1.1 + - azure-mgmt-iotcentral==0.1.0 + - azure-mgmt-iothub==0.5.0 + - azure-mgmt-iothubprovisioningservices==0.2.0 + - azure-mgmt-keyvault==1.1.0 + - azure-mgmt-loganalytics==0.2.0 + - azure-mgmt-logic==3.0.0 + - azure-mgmt-machinelearningcompute==0.4.1 + - azure-mgmt-managementgroups==0.1.0 + - azure-mgmt-managementpartner==0.1.1 + - azure-mgmt-maps==0.1.0 + - azure-mgmt-marketplaceordering==0.1.0 + - azure-mgmt-media==1.0.1 + - azure-mgmt-monitor==0.5.2 + - azure-mgmt-msi==0.2.0 + - azure-mgmt-network==2.7.0 + - azure-mgmt-notificationhubs==2.1.0 + - azure-mgmt-nspkg==3.0.2 + - azure-mgmt-policyinsights==0.1.0 + - azure-mgmt-powerbiembedded==2.0.0 + - azure-mgmt-rdbms==1.9.0 + - azure-mgmt-recoveryservices==0.3.0 + - azure-mgmt-recoveryservicesbackup==0.3.0 + - azure-mgmt-redis==5.0.0 + - azure-mgmt-relay==0.1.0 + - azure-mgmt-reservations==0.2.1 + - azure-mgmt-resource==2.2.0 + - azure-mgmt-scheduler==2.0.0 + - azure-mgmt-search==2.1.0 + - azure-mgmt-servicebus==0.5.3 + - azure-mgmt-servicefabric==0.2.0 + - azure-mgmt-signalr==0.1.1 + - azure-mgmt-sql==0.9.1 + - azure-mgmt-storage==2.0.0 + - azure-mgmt-subscription==0.2.0 + - azure-mgmt-trafficmanager==0.50.0 + - azure-mgmt-web==0.35.0 + - azure-nspkg==3.0.2 + - azure-servicebus==0.21.1 + - azure-servicefabric==6.3.0.0 + - azure-servicemanagement-legacy==0.20.8 + - azure-storage-blob==1.5.0 + - azure-storage-common==1.4.2 + - azure-storage-file==1.4.0 + - azure-storage-queue==1.4.0 + - blinker==1.9.0 + - chardet==5.2.0 + - charset-normalizer==3.4.2 + - click==8.2.1 + - cloudpickle==3.1.1 + - colorlog==6.9.0 + - comm==0.2.2 + - configargparse==1.7.1 + - contourpy==1.3.2 + - cryptography==45.0.4 + - cycler==0.12.1 + - dash==3.1.1 + - decorator==5.2.1 + - diffusers==0.34.0 + - distro==1.9.0 + - einops==0.8.1 + - embreex==2.17.7.post6 + - exceptiongroup==1.3.0 + - executing==2.2.0 + - farama-notifications==0.0.4 + - fasteners==0.19 + - fastjsonschema==2.21.1 + - filelock==3.18.0 + - flask==3.1.1 + - fonttools==4.58.4 + - fsspec==2025.5.1 + - gitdb==4.0.12 + - gitpython==3.1.44 + - gymnasium==0.29.1 + - h11==0.16.0 + - h5py==3.14.0 + - hf-xet==1.1.5 + - httpcore==1.0.9 + - httpx==0.28.1 + - huggingface-hub==0.33.2 + - idna==3.10 + - imageio==2.34.2 + - imageio-ffmpeg==0.6.0 + - importlib-metadata==8.7.0 + - importlib-resources==6.5.2 + - intel-cmplr-lib-ur==2024.2.1 + - intel-openmp==2024.2.1 + - ipython==8.37.0 + - ipywidgets==8.1.7 + - isodate==0.7.2 + - itsdangerous==2.2.0 + - jedi==0.19.2 + - jiter==0.10.0 + - joblib==1.5.1 + - jsonschema==4.24.0 + - jsonschema-specifications==2025.4.1 + - jupyter-core==5.8.1 + - jupyterlab-widgets==3.0.15 + - kiwisolver==1.4.8 + - lazy-loader==0.4 + - lxml==6.0.0 + - manifold3d==3.1.1 + - matplotlib==3.10.3 + - matplotlib-inline==0.1.7 + - mkl==2024.0.0 + - moviepy==2.2.1 + - mplib==0.2.1 + - msal==1.32.3 + - msrest==0.7.1 + - msrestazure==0.6.4.post1 + - narwhals==1.45.0 + - nbformat==5.10.4 + - nest-asyncio==1.6.0 + - networkx==3.4.2 + - numcodecs==0.13.1 + - numpy==1.26.4 + - numpy-quaternion==2024.0.9 + - nvidia-curobo==0.7.7.post1.dev2+dirty + - oauthlib==3.3.1 + - open3d==0.18.0 + - openai==1.93.0 + - opencv-python==4.11.0.86 + - packaging==25.0 + - pandas==2.3.0 + - parso==0.8.4 + - pexpect==4.9.0 + - pillow==11.3.0 + - platformdirs==4.3.8 + - plotly==6.2.0 + - proglog==0.1.12 + - prompt-toolkit==3.0.51 + - protobuf==6.31.1 + - psutil==7.0.0 + - ptyprocess==0.7.0 + - pure-eval==0.2.3 + - pybind11==2.13.6 + - pycollada==0.9.2 + - pycparser==2.22 + - pydantic==2.11.7 + - pydantic-core==2.33.2 + - pyglet==1.5.31 + - pygments==2.19.2 + - pyjwt==2.10.1 + - pyparsing==3.2.3 + - pyperclip==1.9.0 + - pyquaternion==0.9.9 + - python-dateutil==2.9.0.post0 + - python-dotenv==1.1.1 + - pytz==2025.2 + - qwen-vl-utils==0.0.11 + - referencing==0.36.2 + - regex==2024.11.6 + - requests-oauthlib==2.0.0 + - retrying==1.4.0 + - rpds-py==0.26.0 + - rtree==1.4.0 + - safetensors==0.5.3 + - sapien==3.0.0b1 + - scikit-image==0.25.2 + - scikit-learn==1.7.0 + - scipy==1.15.3 + - sentry-sdk==2.32.0 + - setuptools-scm==8.3.1 + - shapely==2.1.1 + - six==1.17.0 + - smmap==5.0.2 + - sniffio==1.3.1 + - stack-data==0.6.3 + - svg-path==6.3 + - sympy==1.14.0 + - tbb==2021.13.1 + - termcolor==3.1.0 + - threadpoolctl==3.6.0 + - tifffile==2025.5.10 + - timm==1.0.16 + - tokenizers==0.21.2 + - tomli==2.2.1 + - toppra==0.6.3 + - tqdm==4.67.1 + - traitlets==5.14.3 + - transformers==4.47.0 + - transforms3d==0.4.2 + - trimesh==4.4.3 + - triton==3.0.0 + - typing-extensions==4.14.0 + - typing-inspection==0.4.1 + - tzdata==2025.2 + - vhacdx==0.0.8.post2 + - wandb==0.21.0 + - warp-lang==1.8.0 + - wcwidth==0.2.13 + - werkzeug==3.1.3 + - widgetsnbextension==4.0.14 + - xatlas==0.0.10 + - xxhash==3.5.0 + - yourdfpy==0.0.58 + - zarr==2.18.3 + - zipp==3.23.0 + diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/Eval_Tiny_DexVLA_requirements.txt b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/Eval_Tiny_DexVLA_requirements.txt new file mode 100644 index 00000000..2863f79f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/Eval_Tiny_DexVLA_requirements.txt @@ -0,0 +1,267 @@ +accelerate==0.26.0 +adal==1.2.7 +addict==2.4.0 +annotated-types==0.7.0 +anyio==4.9.0 +asciitree==0.3.3 +asttokens==3.0.0 +attrs==25.3.0 +av==14.4.0 +azure==4.0.0 +azure-ai-inference==1.0.0b9 +azure-applicationinsights==0.1.1 +azure-batch==4.1.3 +azure-common==1.1.28 +azure-core==1.34.0 +azure-cosmosdb-nspkg==2.0.2 +azure-cosmosdb-table==1.0.6 +azure-datalake-store==0.0.53 +azure-eventgrid==1.3.0 +azure-graphrbac==0.40.0 +azure-keyvault==1.1.0 +azure-loganalytics==0.1.1 +azure-mgmt==4.0.0 +azure-mgmt-advisor==1.0.1 +azure-mgmt-applicationinsights==0.1.1 +azure-mgmt-authorization==0.50.0 +azure-mgmt-batch==5.0.1 +azure-mgmt-batchai==2.0.0 +azure-mgmt-billing==0.2.0 +azure-mgmt-cdn==3.1.0 +azure-mgmt-cognitiveservices==3.0.0 +azure-mgmt-commerce==1.0.1 +azure-mgmt-compute==4.6.2 +azure-mgmt-consumption==2.0.0 +azure-mgmt-containerinstance==1.5.0 +azure-mgmt-containerregistry==2.8.0 +azure-mgmt-containerservice==4.4.0 +azure-mgmt-cosmosdb==0.4.1 +azure-mgmt-datafactory==0.6.0 +azure-mgmt-datalake-analytics==0.6.0 +azure-mgmt-datalake-nspkg==3.0.1 +azure-mgmt-datalake-store==0.5.0 +azure-mgmt-datamigration==1.0.0 +azure-mgmt-devspaces==0.1.0 +azure-mgmt-devtestlabs==2.2.0 +azure-mgmt-dns==2.1.0 +azure-mgmt-eventgrid==1.0.0 +azure-mgmt-eventhub==2.6.0 +azure-mgmt-hanaonazure==0.1.1 +azure-mgmt-iotcentral==0.1.0 +azure-mgmt-iothub==0.5.0 +azure-mgmt-iothubprovisioningservices==0.2.0 +azure-mgmt-keyvault==1.1.0 +azure-mgmt-loganalytics==0.2.0 +azure-mgmt-logic==3.0.0 +azure-mgmt-machinelearningcompute==0.4.1 +azure-mgmt-managementgroups==0.1.0 +azure-mgmt-managementpartner==0.1.1 +azure-mgmt-maps==0.1.0 +azure-mgmt-marketplaceordering==0.1.0 +azure-mgmt-media==1.0.1 +azure-mgmt-monitor==0.5.2 +azure-mgmt-msi==0.2.0 +azure-mgmt-network==2.7.0 +azure-mgmt-notificationhubs==2.1.0 +azure-mgmt-nspkg==3.0.2 +azure-mgmt-policyinsights==0.1.0 +azure-mgmt-powerbiembedded==2.0.0 +azure-mgmt-rdbms==1.9.0 +azure-mgmt-recoveryservices==0.3.0 +azure-mgmt-recoveryservicesbackup==0.3.0 +azure-mgmt-redis==5.0.0 +azure-mgmt-relay==0.1.0 +azure-mgmt-reservations==0.2.1 +azure-mgmt-resource==2.2.0 +azure-mgmt-scheduler==2.0.0 +azure-mgmt-search==2.1.0 +azure-mgmt-servicebus==0.5.3 +azure-mgmt-servicefabric==0.2.0 +azure-mgmt-signalr==0.1.1 +azure-mgmt-sql==0.9.1 +azure-mgmt-storage==2.0.0 +azure-mgmt-subscription==0.2.0 +azure-mgmt-trafficmanager==0.50.0 +azure-mgmt-web==0.35.0 +azure-nspkg==3.0.2 +azure-servicebus==0.21.1 +azure-servicefabric==6.3.0.0 +azure-servicemanagement-legacy==0.20.8 +azure-storage-blob==1.5.0 +azure-storage-common==1.4.2 +azure-storage-file==1.4.0 +azure-storage-queue==1.4.0 +blinker==1.9.0 +brotlicffi @ file:///croot/brotlicffi_1736182461069/work +certifi @ file:///home/conda/feedstock_root/build_artifacts/certifi_1749972191589/work/certifi +cffi @ file:///croot/cffi_1736182485317/work +chamfer==2.0.0 +chardet==5.2.0 +charset-normalizer @ file:///croot/charset-normalizer_1721748349566/work +click==8.2.1 +cloudpickle==3.1.1 +colorlog==6.9.0 +comm==0.2.2 +ConfigArgParse==1.7.1 +contact-graspnet==0.0.0 +contourpy==1.3.2 +cryptography==45.0.4 +cycler==0.12.1 +dash==3.1.1 +decorator==5.2.1 +diffusers==0.34.0 +distro==1.9.0 +einops==0.8.1 +embreex==2.17.7.post6 +emd_ext==0.0.0 +exceptiongroup==1.3.0 +executing==2.2.0 +Farama-Notifications==0.0.4 +fasteners==0.19 +fastjsonschema==2.21.1 +filelock @ file:///croot/filelock_1744281381737/work +Flask==3.1.1 +fonttools==4.58.4 +fsspec==2025.5.1 +gitdb==4.0.12 +GitPython==3.1.44 +gmpy2 @ file:///croot/gmpy2_1738085463648/work +gymnasium==0.29.1 +h11==0.16.0 +h5py==3.14.0 +hf-xet==1.1.5 +httpcore==1.0.9 +httpx==0.28.1 +huggingface-hub==0.33.2 +idna==3.10 +imageio==2.34.2 +imageio-ffmpeg==0.6.0 +importlib_metadata==8.7.0 +importlib_resources==6.5.2 +intel-cmplr-lib-ur==2024.2.1 +intel-openmp==2024.2.1 +ipython==8.37.0 +ipywidgets==8.1.7 +isodate==0.7.2 +itsdangerous==2.2.0 +jedi==0.19.2 +Jinja2 @ file:///croot/jinja2_1741710844255/work +jiter==0.10.0 +joblib==1.5.1 +jsonschema==4.24.0 +jsonschema-specifications==2025.4.1 +jupyter_core==5.8.1 +jupyterlab_widgets==3.0.15 +kiwisolver==1.4.8 +lazy_loader==0.4 +lxml==6.0.0 +manifold3d==3.1.1 +MarkupSafe @ file:///croot/markupsafe_1738584038848/work +matplotlib==3.10.3 +matplotlib-inline==0.1.7 +mkl==2024.0.0 +mkl-service==2.4.0 +mkl_fft @ file:///croot/mkl_fft_1751317925336/work +mkl_random @ file:///croot/mkl_random_1751318527880/work +moviepy==2.2.1 +mplib==0.2.1 +mpmath @ file:///croot/mpmath_1690848262763/work +msal==1.32.3 +msrest==0.7.1 +msrestazure==0.6.4.post1 +narwhals==1.45.0 +nbformat==5.10.4 +nest-asyncio==1.6.0 +networkx @ file:///croot/networkx_1717597493534/work +numcodecs==0.13.1 +numpy==1.26.4 +numpy-quaternion==2024.0.9 +-e git+https://github.com/NVlabs/curobo.git@cca894de9ec74e77a0a4071319c81958991a9108#egg=nvidia_curobo +oauthlib==3.3.1 +open3d==0.18.0 +openai==1.93.0 +opencv-python==4.11.0.86 +packaging==25.0 +pandas==2.3.0 +parso==0.8.4 +pexpect==4.9.0 +pillow @ file:///croot/pillow_1744613067434/work +platformdirs==4.3.8 +plotly==6.2.0 +proglog==0.1.12 +prompt_toolkit==3.0.51 +protobuf==6.31.1 +psutil==7.0.0 +ptyprocess==0.7.0 +pure_eval==0.2.3 +pybind11==2.13.6 +pycollada==0.9.2 +pycparser @ file:///tmp/build/80754af9/pycparser_1636541352034/work +pydantic==2.11.7 +pydantic_core==2.33.2 +pyglet==1.5.31 +Pygments==2.19.2 +PyJWT==2.10.1 +pyparsing==3.2.3 +pyperclip==1.9.0 +pyquaternion==0.9.9 +PySocks @ file:///home/builder/ci_310/pysocks_1640793678128/work +python-dateutil==2.9.0.post0 +python-dotenv==1.1.1 +pytz==2025.2 +PyYAML @ file:///croot/pyyaml_1728657952215/work +qwen-vl-utils==0.0.11 +referencing==0.36.2 +regex==2024.11.6 +requests @ file:///croot/requests_1750426329888/work +requests-oauthlib==2.0.0 +retrying==1.4.0 +rpds-py==0.26.0 +rtree==1.4.0 +safetensors==0.5.3 +sapien==3.0.0b1 +scikit-image==0.25.2 +scikit-learn==1.7.0 +scipy==1.15.3 +sentry-sdk==2.32.0 +setuptools-scm==8.3.1 +shapely==2.1.1 +six==1.17.0 +smmap==5.0.2 +sniffio==1.3.1 +stack-data==0.6.3 +svg.path==6.3 +sympy @ file:///croot/sympy_1738108488918/work +tbb==2021.13.1 +termcolor==3.1.0 +threadpoolctl==3.6.0 +tifffile==2025.5.10 +timm==1.0.16 +tokenizers==0.21.2 +tomli==2.2.1 +toppra==0.6.3 +torch==2.4.1 +torchaudio==2.4.1 +torchvision==0.19.1 +tqdm==4.67.1 +traitlets==5.14.3 +transformers==4.47.0 +transforms3d==0.4.2 +trimesh==4.4.3 +triton==3.0.0 +typing-inspection==0.4.1 +typing_extensions @ file:///croot/typing_extensions_1734714854207/work +tzdata==2025.2 +ur_rtde==1.5.9 +urllib3 @ file:///croot/urllib3_1750775463400/work +vhacdx==0.0.8.post2 +wandb==0.21.0 +warp-lang==1.8.0 +wcwidth==0.2.13 +Werkzeug==3.1.3 +widgetsnbextension==4.0.14 +xatlas==0.0.10 +xxhash==3.5.0 +yourdfpy==0.0.58 +zarr==2.18.3 +zipp==3.23.0 diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/LICENSE b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/LICENSE new file mode 100644 index 00000000..35e5f5e2 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Tony Z. Zhao + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/README.md b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/README.md new file mode 100644 index 00000000..c6a173cd --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/README.md @@ -0,0 +1,115 @@ +# DexVLA (Vision-Language Model with Plug-In Diffusion Expert for Visuomotor Policy Learning) +## Install +To guarantee clean isolation between training and evaluation environments for both DexVLA and TinyVLA, we provide two distinct, self-contained setups.The training and testing environment can be used for both DexVLA and TinyVLA. + +Training Environment: +```bash +cd policy/DexVLA +conda env create -f Train_Tiny_DexVLA_train.yml +conda activate dexvla-robo +cd policy_heads +pip install -e . +``` +Evaluation Environment: + +Follow the RoboTwin 2.0 documentation to set up the RoboTwin environment. Once the environment is activated, run the following commands to install the required packages: +```bash +pip install einops==0.8.1 +pip install transformers==4.47.0 +pip install timm==1.0.16 +pip install diffusers==0.34.0 +pip install qwen-vl-utils==0.0.11 +pip install accelerate==0.26.0 +``` +If you encounter the following error: +```bash +Unrecognized option 'crf'. +Error splitting the argument list: Option not found. +``` +Run the following command to install the required version of ffmpeg: +```bash +conda install -c conda-forge ffmpeg +``` + +## Prepare Training Data + +This step performs data preprocessing, converting the original RoboTwin 2.0 data into the format required for DexVLA training. The `expert_data_num` parameter specifies the number of trajectory pairs to be used as training data. +```bash +python process_data.py ${task_name} ${task_config} ${expert_data_num} +# python process_data.py beat_block_hammer demo_randomized 50 +``` +If success, you will find the data in the `policy/Dexvla/data/sim_${task_name}/${setting}_${expert_data_num}` folder. + +## Train Policy +This step launches the training process. +### Download official Qwen2_VL weights +We construct the VLM backbone by integrating Qwen2-VL-2B.You can download the official weights from this link: + +| Model | Link | +|---------------------|----------------------------------------------------------------| +| Qwen2-VL (~2B) | [huggingface](https://huggingface.co/Qwen/Qwen2-VL-2B-Instruct) | + +**❗❗** After downloading the standard weights, you have to modify the official `config.json` file in the folder. +Please update the 'architectures' field from "Qwen2VLForConditionalGenerationForVLA" to "DexVLA", and change the 'model_type' field from "qwen2_vla" to "dex_vla". +### Download our pretrained ScaleDP-H weights +We released our pretrained weights of ScaleDP-H which is trained after Stage1. Now you can download the weights and directly finetuning your data on Stage 2. + +| Model | Link | +|-------------------|----------------------------------------------------------------| +| ScaleDP-H (~1B) | [huggingface](https://huggingface.co/lesjie/scale_dp_h) | +| ScaleDP-L (~400M) | [huggingface](https://huggingface.co/lesjie/scale_dp_l) | +### Train +The training script are "scripts/aloha/vla_stage2_train.sh". And you need to change following parameters: +1. **OUTPUT** : refers to the save directory for training, which must include the keyword "qwen2" (and optionally "lora"). If LoRA training is used, the name must include "lora" (e.g., "qwen2_lora"). +2. **TASKNAME** : refers to the tasks used for training, which should be corresponded to "your_task_name" in aloha_scripts/constant.py +3. **mnop** : path to the pretrained VLM weights +4. **load_pretrain_dit** : True +5. **DIT_PRETRAIN** :Path to pretrained policy head (ScaleDP). + +Other hyperparameters like "batch_size", "save_steps" could be customized according to your computation resources. + + +Start training by following commands: +```bash +bash ./scripts/aloha/vla_stage2_train.sh +``` + +## Eval Policy +You need to modify the corresponding path in the `deploy_policy.yml` file: +1. **model_path** : Path to the trained model, in the OUTPUT path. +2. **state_path** : Path to `dataset_stats.pkl`, in the OUTPUT path. + +Then execute: +``` +bash eval.sh ${task_name} ${task_config} ${ckpt_setting} ${expert_data_num} ${seed} ${gpu_id} +# bash eval.sh beat_block_hammer demo_randomized 0 50 0 0 +``` + +## Citation + +If you find our works useful for your research and applications, please cite using these BibTeX: + +```bibtex +# DexVLA +@article{wen2025dexvla, + title={DexVLA: Vision-Language Model with Plug-In Diffusion Expert for General Robot Control}, + author={Wen, Junjie and Zhu, Yichen and Li, Jinming and Tang, Zhibin and Shen, Chaomin and Feng, Feifei}, + journal={arXiv preprint arXiv:2502.05855}, + year={2025} +} + +# Diffusion-VLA +@article{wen2024diffusion, + title={Diffusion-VLA: Scaling Robot Foundation Models via Unified Diffusion and Autoregression}, + author={Wen, Junjie and Zhu, Minjie and Zhu, Yichen and Tang, Zhibin and Li, Jinming and Zhou, Zhongyi and Li, Chengmeng and Liu, Xiaoyu and Peng, Yaxin and Shen, Chaomin and others}, + journal={arXiv preprint arXiv:2412.03293}, + year={2024} +} + +# ScaleDP +@article{zhu2024scaling, + title={Scaling diffusion policy in transformer to 1 billion parameters for robotic manipulation}, + author={Zhu, Minjie and Zhu, Yichen and Li, Jinming and Wen, Junjie and Xu, Zhiyuan and Liu, Ning and Cheng, Ran and Shen, Chaomin and Peng, Yaxin and Feng, Feifei and others}, + journal={arXiv preprint arXiv:2409.14411}, + year={2024} +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/Train_Tiny_DexVLA_train.yml b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/Train_Tiny_DexVLA_train.yml new file mode 100644 index 00000000..ce296e7c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/Train_Tiny_DexVLA_train.yml @@ -0,0 +1,215 @@ +name: dexvla-robo +channels: + - defaults +dependencies: + - _libgcc_mutex=0.1 + - _openmp_mutex=5.1 + - bzip2=1.0.8 + - ca-certificates=2024.3.11 + - ld_impl_linux-64=2.38 + - libffi=3.4.4 + - libgcc-ng=11.2.0 + - libgomp=11.2.0 + - libstdcxx-ng=11.2.0 + - libuuid=1.41.5 + - ncurses=6.4 + - openssl=3.0.13 + - pip=24.0 + - python=3.10.14 + - readline=8.2 + - setuptools=69.5.1 + - sqlite=3.45.3 + - tk=8.6.14 + - xz=5.4.6 + - zlib=1.2.13 + - pip: + - absl-py==2.1.0 + - accelerate==1.2.0 + - addict==2.4.0 + - aiofiles==24.1.0 + - aiohttp==3.9.5 + - aiosignal==1.3.1 + - aliyun-python-sdk-core==2.15.1 + - aliyun-python-sdk-kms==2.16.3 + - altair==5.5.0 + - antlr4-python3-runtime==4.9.3 + - anyio==4.9.0 + - asciitree==0.3.3 + - asttokens==3.0.0 + - async-timeout==4.0.3 + - attrs==23.2.0 + - av==14.4.0 + - bitsandbytes==0.41.0 + - certifi==2024.2.2 + - cffi==1.16.0 + - charset-normalizer==3.3.2 + - click==8.2.0 + - conda-pack==0.8.1 + - contourpy==1.2.1 + - crcmod==1.7 + - cryptography==42.0.7 + - cycler==0.12.1 + - datasets==3.2.0 + - decorator==5.2.1 + - deepspeed==0.9.5 + - diffusers==0.33.1 + - dill==0.3.8 + - docker-pycreds==0.4.0 + - draccus==0.11.5 + - einops==0.6.1 + - einops-exts==0.0.4 + - exceptiongroup==1.3.0 + - executing==2.2.0 + - fastapi==0.115.12 + - fasteners==0.19 + - ffmpy==0.5.0 + - filelock==3.14.0 + - flash-attn==2.7.4.post1 + - fonttools==4.51.0 + - frozenlist==1.4.1 + - fsspec==2024.2.0 + - gast==0.5.4 + - gitdb==4.0.12 + - gitpython==3.1.44 + - gradio==3.35.2 + - gradio-client==0.2.9 + - grpcio==1.63.0 + - h11==0.14.0 + - h5py==3.11.0 + - hjson==3.1.0 + - httpcore==0.17.3 + - httpx==0.24.0 + - huggingface-hub==0.31.2 + - hydra-core==1.3.2 + - idna==3.7 + - importlib-metadata==7.1.0 + - ipython==8.36.0 + - jedi==0.19.2 + - jinja2==3.1.4 + - jmespath==0.10.0 + - joblib==1.5.0 + - jsonlines==4.0.0 + - jsonschema==4.23.0 + - jsonschema-specifications==2025.4.1 + - kiwisolver==1.4.5 + - latex2mathml==3.78.0 + - linkify-it-py==2.0.3 + - llvmlite==0.44.0 + - markdown==3.6 + - markdown-it-py==2.2.0 + - markdown2==2.5.3 + - markupsafe==2.1.5 + - matplotlib==3.9.0 + - matplotlib-inline==0.1.7 + - mdit-py-plugins==0.3.3 + - mdurl==0.1.2 + - mergedeep==1.3.4 + - mpmath==1.3.0 + - multidict==6.0.5 + - multiprocess==0.70.16 + - mypy-extensions==1.1.0 + - narwhals==1.39.0 + - networkx==3.3 + - ninja==1.11.1.4 + - numba==0.61.2 + - numcodecs==0.13.1 + - numpy==1.26.4 + - nvidia-cublas-cu12==12.4.5.8 + - nvidia-cuda-cupti-cu12==12.1.105 + - nvidia-cuda-nvrtc-cu12==12.1.105 + - nvidia-cuda-runtime-cu12==12.1.105 + - nvidia-cudnn-cu12==8.9.2.26 + - nvidia-cufft-cu12==11.0.2.54 + - nvidia-cufile-cu12==1.11.1.6 + - nvidia-curand-cu12==10.3.2.106 + - nvidia-cusolver-cu12==11.4.5.107 + - nvidia-cusparse-cu12==12.1.0.106 + - nvidia-cusparselt-cu12==0.6.3 + - nvidia-nccl-cu12==2.20.5 + - nvidia-nvjitlink-cu12==12.6.85 + - nvidia-nvtx-cu12==12.1.105 + - omegaconf==2.3.0 + - opencv-python==4.11.0.86 + - opencv-python-headless==4.11.0.86 + - orjson==3.10.18 + - oss2==2.18.5 + - packaging==24.0 + - pandas==2.2.2 + - parso==0.8.4 + - peft==0.15.2 + - pexpect==4.9.0 + - pillow==10.3.0 + - platformdirs==4.2.2 + - prompt-toolkit==3.0.51 + - protobuf==5.26.1 + - psutil==5.9.8 + - ptyprocess==0.7.0 + - pure-eval==0.2.3 + - py-cpuinfo==9.0.0 + - pyarrow==16.1.0 + - pyarrow-hotfix==0.6 + - pycparser==2.22 + - pycryptodome==3.20.0 + - pydantic==1.10.22 + - pydub==0.25.1 + - pygments==2.19.1 + - pyparsing==3.1.2 + - python-dateutil==2.9.0.post0 + - python-multipart==0.0.20 + - pytz==2024.1 + - pyyaml==6.0.1 + - pyyaml-include==1.4.1 + - qwen-vl-utils==0.0.11 + - referencing==0.36.2 + - regex==2024.5.15 + - requests==2.32.4 + - rpds-py==0.24.0 + - safetensors==0.4.3 + - scikit-learn==1.2.2 + - scipy==1.13.0 + - semantic-version==2.10.0 + - sentencepiece==0.1.99 + - sentry-sdk==2.28.0 + - setproctitle==1.3.6 + - shortuuid==1.0.13 + - simplejson==3.19.2 + - six==1.16.0 + - smmap==5.0.2 + - sniffio==1.3.1 + - sortedcontainers==2.4.0 + - stack-data==0.6.3 + - starlette==0.46.2 + - svgwrite==1.4.3 + - sympy==1.14.0 + - tensorboard==2.16.2 + - tensorboard-data-server==0.7.2 + - threadpoolctl==3.6.0 + - tiktoken==0.7.0 + - timm==0.9.10 + - tokenizers==0.20.3 + - toml==0.10.2 + - tomli==2.0.1 + - torch==2.3.0 + - torchvision==0.18.0 + - tqdm==4.66.4 + - traitlets==5.14.3 + - transformers==4.45.2 + - transformers-stream-generator==0.0.4 + - triton==2.3.0 + - typing-extensions==4.11.0 + - typing-inspect==0.9.0 + - tzdata==2024.1 + - uc-micro-py==1.0.3 + - urllib3==2.2.1 + - uvicorn==0.34.2 + - wandb==0.19.11 + - wavedrom==2.0.3.post3 + - wcwidth==0.2.13 + - websockets==15.0.1 + - werkzeug==3.0.3 + - wheel==0.41.3 + - xxhash==3.4.1 + - yapf==0.40.2 + - yarl==1.9.4 + - zarr==2.18.3 + - zipp==3.18.2 diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/__init__.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/__init__.py new file mode 100644 index 00000000..d4b67709 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/__init__.py @@ -0,0 +1 @@ +from .deploy_policy import * diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/.ipynb_checkpoints/constants-checkpoint.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/.ipynb_checkpoints/constants-checkpoint.py new file mode 100644 index 00000000..6c334deb --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/.ipynb_checkpoints/constants-checkpoint.py @@ -0,0 +1,354 @@ + +# DATA_DIR = './datasets' +DATA_DIR = "/home/jovyan/tzb/h5py_data/" +# DATA_DIR = '/home/jovyan/tzb/h5py_data/' +PRETRAIN_DIR = '/data/team/xuzy/nfs/eai_data/data_WJJ/droid_1dot7t_h5py2' + +TASK_CONFIGS = { + 'folding_data_0609': { + 'dataset_dir': [ + # "/data/efs/qiaoyi/EAI_robot_data/mobile_aloha_3_wheels/20250530_random_fold_stacked_T-shirts_zby_compressed", + # "/data/efs/qiaoyi/EAI_robot_data/mobile_aloha_3_wheels/20250603_random_fold_stacked_T-shirts_zby_2_compressed", + # "/data/efs/qiaoyi/EAI_robot_data/mobile_aloha_3_wheels/20250603_random_fold_stacked_T-shirts_zby_compressed", + "/data/efs/qiaoyi/EAI_robot_data/mobile_aloha_4_wheels/20250521_fold_pants_zby_compressed", + "/data/efs/qiaoyi/EAI_robot_data/mobile_aloha_4_wheels/20250522_fold_pants_zby_compressed", + "/data/efs/qiaoyi/EAI_robot_data/mobile_aloha_4_wheels/20250523_fold_pants_zby_compressed", + "/data/efs/qiaoyi/EAI_robot_data/mobile_aloha_4_wheels/20250526_fold_pants_lyp_compressed", + "/data/efs/qiaoyi/EAI_robot_data/mobile_aloha_4_wheels/20250526_fold_pants_zby_compressed", + "/data/efs/qiaoyi/EAI_robot_data/mobile_aloha_4_wheels/20250527_fold_pants_lyp_compressed", + "/data/efs/qiaoyi/EAI_robot_data/mobile_aloha_4_wheels/20250527_fold_pants_zby_compressed", + # "/data/efs/qiaoyi/EAI_robot_data/mobile_aloha_4_wheels/20250528_fold_T-shirts_zby_compressed", + # "/data/efs/qiaoyi/EAI_robot_data/mobile_aloha_4_wheels/20250529_fold_T-shirts_lyp_compressed", + # "/data/efs/qiaoyi/EAI_robot_data/mobile_aloha_4_wheels/20250529_fold_T-shirts_zby_compressed", + "/data/efs/qiaoyi/EAI_robot_data/static_aloha/20250526_random_folding_pants_Leo_compressed", + "/data/efs/qiaoyi/EAI_robot_data/static_aloha/20250527_random_folding_pants_Leo_compressed", + "/data/efs/qiaoyi/EAI_robot_data/static_aloha/20250528_random_folding_pants_Leo_compressed", + "/data/efs/qiaoyi/EAI_robot_data/static_aloha/20250528_random_folding_pants_zjm_2_compressed", + "/data/efs/qiaoyi/EAI_robot_data/static_aloha/20250528_random_folding_pants_zjm_compressed", + "/data/efs/qiaoyi/EAI_robot_data/static_aloha/20250529_random_folding_pants_Leo_compressed", + "/data/efs/qiaoyi/EAI_robot_data/static_aloha/20250529_random_folding_pants_zjm_2_compressed", + "/data/efs/qiaoyi/EAI_robot_data/static_aloha/20250529_random_folding_pants_zjm_compressed", + "/data/efs/qiaoyi/EAI_robot_data/static_aloha/20250530_random_folding_pants_zjm_compressed", + "/data/efs/qiaoyi/EAI_robot_data/static_aloha/20250603_random_folding_pants_lyp_compressed", + "/data/efs/qiaoyi/EAI_robot_data/static_aloha/20250603_random_folding_pants_zjm_compressed", + # "/data/efs/qiaoyi/EAI_robot_data/static_aloha/folding_shirts_stack_Leo_20250522_compressed", + # "/data/efs/qiaoyi/EAI_robot_data/static_aloha/folding_shirts_stack_zjm_20250522_compressed", + # "/data/efs/qiaoyi/EAI_robot_data/static_aloha/folding_shirts_stack_zjm_20250523_compressed", + "/data/efs/qiaoyi/EAI_robot_data/static_aloha/random_folding_pants_Leo_20250526_noon_compressed", + "/data/efs/qiaoyi/EAI_robot_data/static_aloha/random_folding_pants_zjm_20250526_2_compressed", + "/data/efs/qiaoyi/EAI_robot_data/static_aloha/random_folding_pants_zjm_20250526_compressed", + "/data/efs/qiaoyi/EAI_robot_data/static_aloha/random_folding_pants_zjm_20250527_2_compressed", + "/data/efs/qiaoyi/EAI_robot_data/static_aloha/random_folding_pants_zjm_20250527_compressed" + ], + 'episode_len': 1000, + 'camera_names': ['cam_high', 'cam_left_wrist', 'cam_right_wrist'] + }, + 'folding_blue_shirt': { # for local debug + 'dataset_dir': [ + "/media/rl/HDD/data/data/aloha_data/4_cameras_aloha/folding_shirt" + ], + 'episode_len': 1000, # 1000, + # 'camera_names': ['cam_front', 'cam_high', 'cam_left_wrist', 'cam_right_wrist'] + 'camera_names': ['cam_high', 'cam_left_wrist', 'cam_right_wrist'] + }, + + '3_cameras_random_folding_1_25': { + 'dataset_dir': [ + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_10_extract/folding_second_tshirt_yichen_0108', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_10_extract/folding_second_tshirt_wjj_0108', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_10_extract/folding_random_yichen_0109', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_10_extract/folding_random_table_right_wjj_0109', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_10_extract/folding_basket_two_tshirt_yichen_0109', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_10_extract/folding_basket_second_tshirt_yichen_0110', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_10_extract/folding_basket_second_tshirt_yichen_0109', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_10_extract/folding_basket_second_tshirt_wjj_0110', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/data_01_11_13_7z_exact/data_01_11_13/folding_basket_second_tshirt_yichen_0111', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/data_01_11_13_7z_exact/data_01_11_13/folding_basket_second_tshirt_wjj_0113', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/data_01_11_13_7z_exact/data_01_11_13/folding_basket_second_tshirt_wjj_0111', + + # 1.17 2025 new add + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_15_16_data_extract/weiqing_folding_basket_first_tshirt_dark_blue_yichen_0116", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_15_16_data_extract/weiqing_folding_basket_first_tshirt_pink_wjj_0115", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_15_16_data_extract/weiqing_folding_basket_second_tshirt_blue_yichen_0115", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_15_16_data_extract/weiqing_folding_basket_second_tshirt_dark_blue_yichen_0116", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_15_16_data_extract/weiqing_folding_basket_second_tshirt_red_lxy_0116", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_15_16_data_extract/weiqing_folding_basket_second_tshirt_red_wjj_0116", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_15_16_data_extract/weiqing_folding_basket_second_tshirt_shu_red_yellow_wjj_0116", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_15_16_data_extract/weiqing_folding_basket_second_tshirt_yellow_shu_red_wjj_0116", + + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/1_14_data_move_add_folding_shirt/move_data/folding_basket_second_tshirt_yichen_0114", + + # 1.19 2025 new add + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_18_extract/weiqing_folding_basket_second_dark_blue_shirt_to_polo_lxy_0118", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_17_folding_basket_extract/weiqing_folding_basket_first_yellow_blue_wjj_0117", + # 3 camera views + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_17_folding_basket_extract/weiqing_folding_basket_second_dark_blue_polo_to_blue_shirt_lxy_0117", + # 3 camera views + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_17_folding_basket_extract/weiqing_folding_basket_second_yellow_blue_wjj_0117", + # 3 camera views + + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/1_21_7z_extract/folding_random_short_first_wjj_0121", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/1_21_7z_extract/folding_random_short_second_wjj_0121", + + # 1.23 + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/1_22_7z_extract/folding_random_short_second_wjj_0122", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/1_22_7z_extract/folding_random_short_first_wjj_0122", + # 1.25 add + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/1_24_folding_7z_extract/folding_random_tshirt_first_wjj_0124", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/1_24_folding_7z_extract/folding_random_tshirt_second_wjj_0124", + ], + 'episode_len': 1000, # 1000, + # 'camera_names': ['cam_high', 'cam_low', 'cam_left_wrist', 'cam_right_wrist'] + 'camera_names': ['cam_high', 'cam_left_wrist', 'cam_right_wrist'] + }, + + '3_cameras_all_data_1_17': { + 'dataset_dir': [ + + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/fold_shirt_lxy1213', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/fold_shirt_lxy1214', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/fold_shirt_zmj1212', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/fold_shirt_zmj1213', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/fold_shirt_zzy1213', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_junjie_1224', # 50 + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_zhongyi_1224', # 42 + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/fold_shirt_wjj1213_meeting_room', # 42 + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_shirt_12_30_12_31_extract/folding_shirt_12_30_12_31/folding_shirt_12_30_wjj_weiqing_recover', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_shirt_12_30_12_31_extract/folding_shirt_12_30_12_31/folding_shirt_12_31_wjj_lab_marble_recover', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_shirt_12_30_12_31_extract/folding_shirt_12_30_12_31/folding_shirt_12_31_zhouzy_lab_marble', + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_blue_tshirt_yichen_0103", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_blue_tshirt_xiaoyu_0103", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_blue_tshirt_yichen_0102", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_shirt_12_28_zzy_right_first", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_shirt_12_27_office", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/0107_wjj_folding_blue_shirt", + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_10_extract/folding_second_tshirt_yichen_0108', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_10_extract/folding_second_tshirt_wjj_0108', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_10_extract/folding_random_yichen_0109', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_10_extract/folding_random_table_right_wjj_0109', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_10_extract/folding_basket_two_tshirt_yichen_0109', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_10_extract/folding_basket_second_tshirt_yichen_0110', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_10_extract/folding_basket_second_tshirt_yichen_0109', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_10_extract/folding_basket_second_tshirt_wjj_0110', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/data_01_11_13_7z_exact/data_01_11_13/folding_basket_second_tshirt_yichen_0111', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/data_01_11_13_7z_exact/data_01_11_13/folding_basket_second_tshirt_wjj_0113', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/data_01_11_13_7z_exact/data_01_11_13/folding_basket_second_tshirt_wjj_0111', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/1_14_data_move_add_folding_shirt/move_data/folding_basket_second_tshirt_yichen_0114', + # 1.17 2025 new add + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_15_16_data_extract/weiqing_folding_basket_first_tshirt_dark_blue_yichen_0116", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_15_16_data_extract/weiqing_folding_basket_first_tshirt_pink_wjj_0115", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_15_16_data_extract/weiqing_folding_basket_second_tshirt_blue_yichen_0115", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_15_16_data_extract/weiqing_folding_basket_second_tshirt_dark_blue_yichen_0116", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_15_16_data_extract/weiqing_folding_basket_second_tshirt_red_lxy_0116", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_15_16_data_extract/weiqing_folding_basket_second_tshirt_red_wjj_0116", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_15_16_data_extract/weiqing_folding_basket_second_tshirt_shu_red_yellow_wjj_0116", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_15_16_data_extract/weiqing_folding_basket_second_tshirt_yellow_shu_red_wjj_0116", + + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/clean_table_ljm_1217', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/clean_table_zmj_1217_green_plate_coke_can_brown_mug_bottle', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/clean_table_lxy_1220_blue_plate_pink_paper_cup_plastic_bag_knife', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/clean_table_zzy_1220_green_paper_cup_wulong_bottle_pink_bowl_brown_spoon', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/clean_table_zmj_1220_green_cup_blue_paper_ball_pink_plate_sprite', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/clean_table_zmj_1217_green_plate_coke_can_brown_mug_bottle', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/clean_table_lxy_1222_pick_place_water_left_arm', + + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/aloha_data/pick_cup_and_pour_water_wjj_weiqing_coke', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/aloha_data/pick_cars_from_moving_belt_waibao_1227', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/aloha_data/pick_cup_and_pour_water_wjj_weiqing_coffee', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/aloha_data/pick_cars_from_moving_belt_zhumj_1227', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/aloha_data/hang_cups_waibao', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/aloha_data/storage_bottle_green_tea_oolong_mineral_water_ljm_weiqing_1225_right_hand', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/aloha_data/storage_bottle_green_tea_oolong_mineral_water_lxy_weiqing_1225', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/get_papercup_yichen_1223', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/pour_coffee_zhaopeiting_1224', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/get_papercup_and_pour_coke_yichen_1224', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/pick_up_coke_in_refrigerator_yichen_1223', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/pour_rice_yichen_0102', + + # from Shanghai University + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/pick_paper_ball_from_bike', + + ], + 'episode_len': 1000, # 1000, + # 'camera_names': ['cam_high', 'cam_low', 'cam_left_wrist', 'cam_right_wrist'] + 'camera_names': ['cam_high', 'cam_left_wrist', 'cam_right_wrist'] + }, + + '3_cameras_1_17_standard_folding': { + 'dataset_dir': [ + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/fold_shirt_lxy1213', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/fold_shirt_lxy1214', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/fold_shirt_zmj1212', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/fold_shirt_zmj1213', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/fold_shirt_zzy1213', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_junjie_1224', # 50 + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_zhongyi_1224', # 42 + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/fold_shirt_wjj1213_meeting_room', # 42 + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_shirt_12_30_12_31_extract/folding_shirt_12_30_12_31/folding_shirt_12_30_wjj_weiqing_recover', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_shirt_12_30_12_31_extract/folding_shirt_12_30_12_31/folding_shirt_12_31_wjj_lab_marble_recover', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_shirt_12_30_12_31_extract/folding_shirt_12_30_12_31/folding_shirt_12_31_zhouzy_lab_marble', + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_blue_tshirt_yichen_0103", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_blue_tshirt_xiaoyu_0103", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_blue_tshirt_yichen_0102", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_shirt_12_28_zzy_right_first", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_shirt_12_27_office", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/0107_wjj_folding_blue_shirt", + ], + 'episode_len': 1000, # 1000, + # 'camera_names': ['cam_high', 'cam_low', 'cam_left_wrist', 'cam_right_wrist'] + 'camera_names': ['cam_high', 'cam_left_wrist', 'cam_right_wrist'] + }, + + '3_cameras_all_data_1_25': { + 'dataset_dir': [ + + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/fold_shirt_lxy1213', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/fold_shirt_lxy1214', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/fold_shirt_zmj1212', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/fold_shirt_zmj1213', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/fold_shirt_zzy1213', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_junjie_1224', # 50 + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_zhongyi_1224', # 42 + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/fold_shirt_wjj1213_meeting_room', # 42 + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_shirt_12_30_12_31_extract/folding_shirt_12_30_12_31/folding_shirt_12_30_wjj_weiqing_recover', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_shirt_12_30_12_31_extract/folding_shirt_12_30_12_31/folding_shirt_12_31_wjj_lab_marble_recover', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_shirt_12_30_12_31_extract/folding_shirt_12_30_12_31/folding_shirt_12_31_zhouzy_lab_marble', + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_blue_tshirt_yichen_0103", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_blue_tshirt_xiaoyu_0103", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_blue_tshirt_yichen_0102", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_shirt_12_28_zzy_right_first", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/folding_shirt_12_27_office", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/0107_wjj_folding_blue_shirt", + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_10_extract/folding_second_tshirt_yichen_0108', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_10_extract/folding_second_tshirt_wjj_0108', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_10_extract/folding_random_yichen_0109', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_10_extract/folding_random_table_right_wjj_0109', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_10_extract/folding_basket_two_tshirt_yichen_0109', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_10_extract/folding_basket_second_tshirt_yichen_0110', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_10_extract/folding_basket_second_tshirt_yichen_0109', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_10_extract/folding_basket_second_tshirt_wjj_0110', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/data_01_11_13_7z_exact/data_01_11_13/folding_basket_second_tshirt_yichen_0111', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/data_01_11_13_7z_exact/data_01_11_13/folding_basket_second_tshirt_wjj_0113', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/data_01_11_13_7z_exact/data_01_11_13/folding_basket_second_tshirt_wjj_0111', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/1_14_data_move_add_folding_shirt/move_data/folding_basket_second_tshirt_yichen_0114', + # 1.17 2025 new add + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_15_16_data_extract/weiqing_folding_basket_first_tshirt_dark_blue_yichen_0116", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_15_16_data_extract/weiqing_folding_basket_first_tshirt_pink_wjj_0115", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_15_16_data_extract/weiqing_folding_basket_second_tshirt_blue_yichen_0115", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_15_16_data_extract/weiqing_folding_basket_second_tshirt_dark_blue_yichen_0116", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_15_16_data_extract/weiqing_folding_basket_second_tshirt_red_lxy_0116", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_15_16_data_extract/weiqing_folding_basket_second_tshirt_red_wjj_0116", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_15_16_data_extract/weiqing_folding_basket_second_tshirt_shu_red_yellow_wjj_0116", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_15_16_data_extract/weiqing_folding_basket_second_tshirt_yellow_shu_red_wjj_0116", + + # 1.21 added + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_20_data_extract/unloading_dryer_yichen_0120", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_20_data_extract/unloading_dryer_yichen_0119", + + # 1.22 + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/1_21_7z_extract/folding_random_short_first_wjj_0121", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/1_21_7z_extract/folding_random_short_second_wjj_0121", + + # 1.23 + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/1_22_7z_extract/folding_random_short_second_wjj_0122", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/1_22_7z_extract/folding_random_short_first_wjj_0122", + + # 1.25 + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/1_24_folding_7z_extract/folding_random_tshirt_first_wjj_0124", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/1_24_folding_7z_extract/folding_random_tshirt_second_wjj_0124", + + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/1_24_7z_extract/truncate_push_basket_to_left_1_24/", + + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/clean_table_ljm_1217', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/clean_table_zmj_1217_green_plate_coke_can_brown_mug_bottle', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/clean_table_lxy_1220_blue_plate_pink_paper_cup_plastic_bag_knife', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/clean_table_zzy_1220_green_paper_cup_wulong_bottle_pink_bowl_brown_spoon', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/clean_table_zmj_1220_green_cup_blue_paper_ball_pink_plate_sprite', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/clean_table_zmj_1217_green_plate_coke_can_brown_mug_bottle', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/clean_table_lxy_1222_pick_place_water_left_arm', + + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/aloha_data/pick_cup_and_pour_water_wjj_weiqing_coke', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/aloha_data/pick_cars_from_moving_belt_waibao_1227', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/aloha_data/pick_cup_and_pour_water_wjj_weiqing_coffee', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/aloha_data/pick_cars_from_moving_belt_zhumj_1227', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/aloha_data/hang_cups_waibao', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/aloha_data/storage_bottle_green_tea_oolong_mineral_water_ljm_weiqing_1225_right_hand', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/aloha_data/storage_bottle_green_tea_oolong_mineral_water_lxy_weiqing_1225', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/get_papercup_yichen_1223', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/pour_coffee_zhaopeiting_1224', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/get_papercup_and_pour_coke_yichen_1224', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/pick_up_coke_in_refrigerator_yichen_1223', + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/pour_rice_yichen_0102', + + # from Shanghai University + '/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/pick_paper_ball_from_bike', + + ], + 'episode_len': 1000, # 1000, + # 'camera_names': ['cam_front', 'cam_high', 'cam_left_wrist', 'cam_right_wrist'] + 'camera_names': ['cam_high', 'cam_left_wrist', 'cam_right_wrist'] + }, + + '3_cameras_only_unloading_dryer': { + 'dataset_dir': [ + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_20_data_extract/unloading_dryer_yichen_0120", + "/home/jovyan/tzb/h5py_data/aloha_bimanual/aloha_4views/7z_1_20_data_extract/unloading_dryer_yichen_0119", + ], + 'episode_len': 1000, # 1000, + # 'camera_names': ['cam_front', 'cam_high', 'cam_left_wrist', 'cam_right_wrist'] + 'camera_names': ['cam_high', 'cam_left_wrist', 'cam_right_wrist'] + }, +} + +### ALOHA fixed constants +DT = 0.02 +JOINT_NAMES = ["waist", "shoulder", "elbow", "forearm_roll", "wrist_angle", "wrist_rotate"] +START_ARM_POSE = [0, -0.96, 1.16, 0, -0.3, 0, 0.02239, -0.02239, 0, -0.96, 1.16, 0, -0.3, 0, 0.02239, -0.02239] +FPS = 50 +# Left finger position limits (qpos[7]), right_finger = -1 * left_finger +MASTER_GRIPPER_POSITION_OPEN = 0.02417 +MASTER_GRIPPER_POSITION_CLOSE = 0.01244 +PUPPET_GRIPPER_POSITION_OPEN = 0.05800 +PUPPET_GRIPPER_POSITION_CLOSE = 0.01844 + +# Gripper joint limits (qpos[6]) +MASTER_GRIPPER_JOINT_OPEN = 0.3083 +MASTER_GRIPPER_JOINT_CLOSE = -0.6842 +PUPPET_GRIPPER_JOINT_OPEN = 1.4910 +PUPPET_GRIPPER_JOINT_CLOSE = -0.6213 + +############################ Helper functions ############################ + +MASTER_GRIPPER_POSITION_NORMALIZE_FN = lambda x: (x - MASTER_GRIPPER_POSITION_CLOSE) / \ + (MASTER_GRIPPER_POSITION_OPEN - MASTER_GRIPPER_POSITION_CLOSE) +PUPPET_GRIPPER_POSITION_NORMALIZE_FN = lambda x: (x - PUPPET_GRIPPER_POSITION_CLOSE) / ( + PUPPET_GRIPPER_POSITION_OPEN - PUPPET_GRIPPER_POSITION_CLOSE) +MASTER_GRIPPER_POSITION_UNNORMALIZE_FN = lambda x: x * ( + MASTER_GRIPPER_POSITION_OPEN - MASTER_GRIPPER_POSITION_CLOSE) + MASTER_GRIPPER_POSITION_CLOSE +PUPPET_GRIPPER_POSITION_UNNORMALIZE_FN = lambda x: x * ( + PUPPET_GRIPPER_POSITION_OPEN - PUPPET_GRIPPER_POSITION_CLOSE) + PUPPET_GRIPPER_POSITION_CLOSE +MASTER2PUPPET_POSITION_FN = lambda x: PUPPET_GRIPPER_POSITION_UNNORMALIZE_FN(MASTER_GRIPPER_POSITION_NORMALIZE_FN(x)) + +MASTER_GRIPPER_JOINT_NORMALIZE_FN = lambda x: (x - MASTER_GRIPPER_JOINT_CLOSE) / ( + MASTER_GRIPPER_JOINT_OPEN - MASTER_GRIPPER_JOINT_CLOSE) +PUPPET_GRIPPER_JOINT_NORMALIZE_FN = lambda x: (x - PUPPET_GRIPPER_JOINT_CLOSE) / ( + PUPPET_GRIPPER_JOINT_OPEN - PUPPET_GRIPPER_JOINT_CLOSE) +MASTER_GRIPPER_JOINT_UNNORMALIZE_FN = lambda x: x * ( + MASTER_GRIPPER_JOINT_OPEN - MASTER_GRIPPER_JOINT_CLOSE) + MASTER_GRIPPER_JOINT_CLOSE +PUPPET_GRIPPER_JOINT_UNNORMALIZE_FN = lambda x: x * ( + PUPPET_GRIPPER_JOINT_OPEN - PUPPET_GRIPPER_JOINT_CLOSE) + PUPPET_GRIPPER_JOINT_CLOSE +MASTER2PUPPET_JOINT_FN = lambda x: PUPPET_GRIPPER_JOINT_UNNORMALIZE_FN(MASTER_GRIPPER_JOINT_NORMALIZE_FN(x)) + +MASTER_GRIPPER_VELOCITY_NORMALIZE_FN = lambda x: x / (MASTER_GRIPPER_POSITION_OPEN - MASTER_GRIPPER_POSITION_CLOSE) +PUPPET_GRIPPER_VELOCITY_NORMALIZE_FN = lambda x: x / (PUPPET_GRIPPER_POSITION_OPEN - PUPPET_GRIPPER_POSITION_CLOSE) + +MASTER_POS2JOINT = lambda x: MASTER_GRIPPER_POSITION_NORMALIZE_FN(x) * ( + MASTER_GRIPPER_JOINT_OPEN - MASTER_GRIPPER_JOINT_CLOSE) + MASTER_GRIPPER_JOINT_CLOSE +MASTER_JOINT2POS = lambda x: MASTER_GRIPPER_POSITION_UNNORMALIZE_FN( + (x - MASTER_GRIPPER_JOINT_CLOSE) / (MASTER_GRIPPER_JOINT_OPEN - MASTER_GRIPPER_JOINT_CLOSE)) +PUPPET_POS2JOINT = lambda x: PUPPET_GRIPPER_POSITION_NORMALIZE_FN(x) * ( + PUPPET_GRIPPER_JOINT_OPEN - PUPPET_GRIPPER_JOINT_CLOSE) + PUPPET_GRIPPER_JOINT_CLOSE +PUPPET_JOINT2POS = lambda x: PUPPET_GRIPPER_POSITION_UNNORMALIZE_FN( + (x - PUPPET_GRIPPER_JOINT_CLOSE) / (PUPPET_GRIPPER_JOINT_OPEN - PUPPET_GRIPPER_JOINT_CLOSE)) + +MASTER_GRIPPER_JOINT_MID = (MASTER_GRIPPER_JOINT_OPEN + MASTER_GRIPPER_JOINT_CLOSE) / 2 diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/__init__.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/__init__.py new file mode 100644 index 00000000..7a9b492d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/__init__.py @@ -0,0 +1 @@ +from .lerobot_constants import * \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/auto_record.sh b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/auto_record.sh new file mode 100644 index 00000000..cc8748b7 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/auto_record.sh @@ -0,0 +1,15 @@ +if [ "$2" -lt 0 ]; then + echo "# of episodes not valid" + exit +fi + +echo "Task: $1" +for (( i=0; i<$2; i++ )) +do + echo "Starting episode $i" + python3 record_episodes.py --task "$1" + if [ $? -ne 0 ]; then + echo "Failed to execute command. Returning" + exit + fi +done \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/constants.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/constants.py new file mode 100644 index 00000000..2482bd42 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/constants.py @@ -0,0 +1,66 @@ + +# DATA_DIR = './datasets' +DATA_DIR = "/home/h5py_data/" +PRETRAIN_DIR = '/data/team/' + +TASK_CONFIGS = { + "robotwin_task_name": { + 'dataset_dir': [DATA_DIR + "sim-/aloha-agilex-1-m1_b1_l1_h0.03_c0_D435-100"], + 'episode_len': 500, + 'camera_names': ['cam_high', 'cam_left_wrist', 'cam_right_wrist'], + "sample_weights": [1, 1] + }, +} + +### ALOHA fixed constants +DT = 0.02 +JOINT_NAMES = ["waist", "shoulder", "elbow", "forearm_roll", "wrist_angle", "wrist_rotate"] +START_ARM_POSE = [0, -0.96, 1.16, 0, -0.3, 0, 0.02239, -0.02239, 0, -0.96, 1.16, 0, -0.3, 0, 0.02239, -0.02239] +FPS = 50 +# Left finger position limits (qpos[7]), right_finger = -1 * left_finger +MASTER_GRIPPER_POSITION_OPEN = 0.02417 +MASTER_GRIPPER_POSITION_CLOSE = 0.01244 +PUPPET_GRIPPER_POSITION_OPEN = 0.05800 +PUPPET_GRIPPER_POSITION_CLOSE = 0.01844 + +# Gripper joint limits (qpos[6]) +MASTER_GRIPPER_JOINT_OPEN = 0.3083 +MASTER_GRIPPER_JOINT_CLOSE = -0.6842 +PUPPET_GRIPPER_JOINT_OPEN = 1.4910 +PUPPET_GRIPPER_JOINT_CLOSE = -0.6213 + +############################ Helper functions ############################ + +MASTER_GRIPPER_POSITION_NORMALIZE_FN = lambda x: (x - MASTER_GRIPPER_POSITION_CLOSE) / \ + (MASTER_GRIPPER_POSITION_OPEN - MASTER_GRIPPER_POSITION_CLOSE) +PUPPET_GRIPPER_POSITION_NORMALIZE_FN = lambda x: (x - PUPPET_GRIPPER_POSITION_CLOSE) / ( + PUPPET_GRIPPER_POSITION_OPEN - PUPPET_GRIPPER_POSITION_CLOSE) +MASTER_GRIPPER_POSITION_UNNORMALIZE_FN = lambda x: x * ( + MASTER_GRIPPER_POSITION_OPEN - MASTER_GRIPPER_POSITION_CLOSE) + MASTER_GRIPPER_POSITION_CLOSE +PUPPET_GRIPPER_POSITION_UNNORMALIZE_FN = lambda x: x * ( + PUPPET_GRIPPER_POSITION_OPEN - PUPPET_GRIPPER_POSITION_CLOSE) + PUPPET_GRIPPER_POSITION_CLOSE +MASTER2PUPPET_POSITION_FN = lambda x: PUPPET_GRIPPER_POSITION_UNNORMALIZE_FN(MASTER_GRIPPER_POSITION_NORMALIZE_FN(x)) + +MASTER_GRIPPER_JOINT_NORMALIZE_FN = lambda x: (x - MASTER_GRIPPER_JOINT_CLOSE) / ( + MASTER_GRIPPER_JOINT_OPEN - MASTER_GRIPPER_JOINT_CLOSE) +PUPPET_GRIPPER_JOINT_NORMALIZE_FN = lambda x: (x - PUPPET_GRIPPER_JOINT_CLOSE) / ( + PUPPET_GRIPPER_JOINT_OPEN - PUPPET_GRIPPER_JOINT_CLOSE) +MASTER_GRIPPER_JOINT_UNNORMALIZE_FN = lambda x: x * ( + MASTER_GRIPPER_JOINT_OPEN - MASTER_GRIPPER_JOINT_CLOSE) + MASTER_GRIPPER_JOINT_CLOSE +PUPPET_GRIPPER_JOINT_UNNORMALIZE_FN = lambda x: x * ( + PUPPET_GRIPPER_JOINT_OPEN - PUPPET_GRIPPER_JOINT_CLOSE) + PUPPET_GRIPPER_JOINT_CLOSE +MASTER2PUPPET_JOINT_FN = lambda x: PUPPET_GRIPPER_JOINT_UNNORMALIZE_FN(MASTER_GRIPPER_JOINT_NORMALIZE_FN(x)) + +MASTER_GRIPPER_VELOCITY_NORMALIZE_FN = lambda x: x / (MASTER_GRIPPER_POSITION_OPEN - MASTER_GRIPPER_POSITION_CLOSE) +PUPPET_GRIPPER_VELOCITY_NORMALIZE_FN = lambda x: x / (PUPPET_GRIPPER_POSITION_OPEN - PUPPET_GRIPPER_POSITION_CLOSE) + +MASTER_POS2JOINT = lambda x: MASTER_GRIPPER_POSITION_NORMALIZE_FN(x) * ( + MASTER_GRIPPER_JOINT_OPEN - MASTER_GRIPPER_JOINT_CLOSE) + MASTER_GRIPPER_JOINT_CLOSE +MASTER_JOINT2POS = lambda x: MASTER_GRIPPER_POSITION_UNNORMALIZE_FN( + (x - MASTER_GRIPPER_JOINT_CLOSE) / (MASTER_GRIPPER_JOINT_OPEN - MASTER_GRIPPER_JOINT_CLOSE)) +PUPPET_POS2JOINT = lambda x: PUPPET_GRIPPER_POSITION_NORMALIZE_FN(x) * ( + PUPPET_GRIPPER_JOINT_OPEN - PUPPET_GRIPPER_JOINT_CLOSE) + PUPPET_GRIPPER_JOINT_CLOSE +PUPPET_JOINT2POS = lambda x: PUPPET_GRIPPER_POSITION_UNNORMALIZE_FN( + (x - PUPPET_GRIPPER_JOINT_CLOSE) / (PUPPET_GRIPPER_JOINT_OPEN - PUPPET_GRIPPER_JOINT_CLOSE)) + +MASTER_GRIPPER_JOINT_MID = (MASTER_GRIPPER_JOINT_OPEN + MASTER_GRIPPER_JOINT_CLOSE) / 2 diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/lerobot_constants.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/lerobot_constants.py new file mode 100644 index 00000000..d1d36dc3 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/lerobot_constants.py @@ -0,0 +1,54 @@ + + +TASK_CONFIGS = { + 'A': { + 'dataset_dir': [ + 'A_0103', + 'A_0102', + ], + 'episode_len': 2000, # 1000, + 'camera_names': ['observation.images.cam_high', + "observation.images.cam_left_wrist", "observation.images.cam_right_wrist"] + }, +} + +### ALOHA fixed constants +DT = 0.02 +JOINT_NAMES = ["waist", "shoulder", "elbow", "forearm_roll", "wrist_angle", "wrist_rotate"] +START_ARM_POSE = [0, -0.96, 1.16, 0, -0.3, 0, 0.02239, -0.02239, 0, -0.96, 1.16, 0, -0.3, 0, 0.02239, -0.02239] +FPS = 50 +# Left finger position limits (qpos[7]), right_finger = -1 * left_finger +MASTER_GRIPPER_POSITION_OPEN = 0.02417 +MASTER_GRIPPER_POSITION_CLOSE = 0.01244 +PUPPET_GRIPPER_POSITION_OPEN = 0.05800 +PUPPET_GRIPPER_POSITION_CLOSE = 0.01844 + +# Gripper joint limits (qpos[6]) +MASTER_GRIPPER_JOINT_OPEN = 0.3083 +MASTER_GRIPPER_JOINT_CLOSE = -0.6842 +PUPPET_GRIPPER_JOINT_OPEN = 1.4910 +PUPPET_GRIPPER_JOINT_CLOSE = -0.6213 + +############################ Helper functions ############################ + +MASTER_GRIPPER_POSITION_NORMALIZE_FN = lambda x: (x - MASTER_GRIPPER_POSITION_CLOSE) / (MASTER_GRIPPER_POSITION_OPEN - MASTER_GRIPPER_POSITION_CLOSE) +PUPPET_GRIPPER_POSITION_NORMALIZE_FN = lambda x: (x - PUPPET_GRIPPER_POSITION_CLOSE) / (PUPPET_GRIPPER_POSITION_OPEN - PUPPET_GRIPPER_POSITION_CLOSE) +MASTER_GRIPPER_POSITION_UNNORMALIZE_FN = lambda x: x * (MASTER_GRIPPER_POSITION_OPEN - MASTER_GRIPPER_POSITION_CLOSE) + MASTER_GRIPPER_POSITION_CLOSE +PUPPET_GRIPPER_POSITION_UNNORMALIZE_FN = lambda x: x * (PUPPET_GRIPPER_POSITION_OPEN - PUPPET_GRIPPER_POSITION_CLOSE) + PUPPET_GRIPPER_POSITION_CLOSE +MASTER2PUPPET_POSITION_FN = lambda x: PUPPET_GRIPPER_POSITION_UNNORMALIZE_FN(MASTER_GRIPPER_POSITION_NORMALIZE_FN(x)) + +MASTER_GRIPPER_JOINT_NORMALIZE_FN = lambda x: (x - MASTER_GRIPPER_JOINT_CLOSE) / (MASTER_GRIPPER_JOINT_OPEN - MASTER_GRIPPER_JOINT_CLOSE) +PUPPET_GRIPPER_JOINT_NORMALIZE_FN = lambda x: (x - PUPPET_GRIPPER_JOINT_CLOSE) / (PUPPET_GRIPPER_JOINT_OPEN - PUPPET_GRIPPER_JOINT_CLOSE) +MASTER_GRIPPER_JOINT_UNNORMALIZE_FN = lambda x: x * (MASTER_GRIPPER_JOINT_OPEN - MASTER_GRIPPER_JOINT_CLOSE) + MASTER_GRIPPER_JOINT_CLOSE +PUPPET_GRIPPER_JOINT_UNNORMALIZE_FN = lambda x: x * (PUPPET_GRIPPER_JOINT_OPEN - PUPPET_GRIPPER_JOINT_CLOSE) + PUPPET_GRIPPER_JOINT_CLOSE +MASTER2PUPPET_JOINT_FN = lambda x: PUPPET_GRIPPER_JOINT_UNNORMALIZE_FN(MASTER_GRIPPER_JOINT_NORMALIZE_FN(x)) + +MASTER_GRIPPER_VELOCITY_NORMALIZE_FN = lambda x: x / (MASTER_GRIPPER_POSITION_OPEN - MASTER_GRIPPER_POSITION_CLOSE) +PUPPET_GRIPPER_VELOCITY_NORMALIZE_FN = lambda x: x / (PUPPET_GRIPPER_POSITION_OPEN - PUPPET_GRIPPER_POSITION_CLOSE) + +MASTER_POS2JOINT = lambda x: MASTER_GRIPPER_POSITION_NORMALIZE_FN(x) * (MASTER_GRIPPER_JOINT_OPEN - MASTER_GRIPPER_JOINT_CLOSE) + MASTER_GRIPPER_JOINT_CLOSE +MASTER_JOINT2POS = lambda x: MASTER_GRIPPER_POSITION_UNNORMALIZE_FN((x - MASTER_GRIPPER_JOINT_CLOSE) / (MASTER_GRIPPER_JOINT_OPEN - MASTER_GRIPPER_JOINT_CLOSE)) +PUPPET_POS2JOINT = lambda x: PUPPET_GRIPPER_POSITION_NORMALIZE_FN(x) * (PUPPET_GRIPPER_JOINT_OPEN - PUPPET_GRIPPER_JOINT_CLOSE) + PUPPET_GRIPPER_JOINT_CLOSE +PUPPET_JOINT2POS = lambda x: PUPPET_GRIPPER_POSITION_UNNORMALIZE_FN((x - PUPPET_GRIPPER_JOINT_CLOSE) / (PUPPET_GRIPPER_JOINT_OPEN - PUPPET_GRIPPER_JOINT_CLOSE)) + +MASTER_GRIPPER_JOINT_MID = (MASTER_GRIPPER_JOINT_OPEN + MASTER_GRIPPER_JOINT_CLOSE)/2 diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/one_side_teleop.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/one_side_teleop.py new file mode 100644 index 00000000..ccdf54f9 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/one_side_teleop.py @@ -0,0 +1,70 @@ +import time +import sys +import IPython +e = IPython.embed + +from interbotix_xs_modules.arm import InterbotixManipulatorXS +from interbotix_xs_msgs.msg import JointSingleCommand +from lerobot_constants import MASTER2PUPPET_JOINT_FN, DT, START_ARM_POSE, MASTER_GRIPPER_JOINT_MID, PUPPET_GRIPPER_JOINT_CLOSE +from robot_utils import torque_on, torque_off, move_arms, move_grippers, get_arm_gripper_positions + +def prep_robots(master_bot, puppet_bot): + # reboot gripper motors, and set operating modes for all motors + puppet_bot.dxl.robot_reboot_motors("single", "gripper", True) + puppet_bot.dxl.robot_set_operating_modes("group", "arm", "position") + puppet_bot.dxl.robot_set_operating_modes("single", "gripper", "current_based_position") + master_bot.dxl.robot_set_operating_modes("group", "arm", "position") + master_bot.dxl.robot_set_operating_modes("single", "gripper", "position") + # puppet_bot.dxl.robot_set_motor_registers("single", "gripper", 'current_limit', 1000) # TODO(tonyzhaozh) figure out how to set this limit + torque_on(puppet_bot) + torque_on(master_bot) + + # move arms to starting position + start_arm_qpos = START_ARM_POSE[:6] + move_arms([master_bot, puppet_bot], [start_arm_qpos] * 2, move_time=1) + # move grippers to starting position + move_grippers([master_bot, puppet_bot], [MASTER_GRIPPER_JOINT_MID, PUPPET_GRIPPER_JOINT_CLOSE], move_time=0.5) + + +def press_to_start(master_bot): + # press gripper to start data collection + # disable torque for only gripper joint of master robot to allow user movement + master_bot.dxl.robot_torque_enable("single", "gripper", False) + print(f'Close the gripper to start') + close_thresh = -0.3 + pressed = False + while not pressed: + gripper_pos = get_arm_gripper_positions(master_bot) + if gripper_pos < close_thresh: + pressed = True + time.sleep(DT/10) + torque_off(master_bot) + print(f'Started!') + + +def teleop(robot_side): + """ A standalone function for experimenting with teleoperation. No data recording. """ + puppet_bot = InterbotixManipulatorXS(robot_model="vx300s", group_name="arm", gripper_name="gripper", robot_name=f'puppet_{robot_side}', init_node=True) + master_bot = InterbotixManipulatorXS(robot_model="wx250s", group_name="arm", gripper_name="gripper", robot_name=f'master_{robot_side}', init_node=False) + + prep_robots(master_bot, puppet_bot) + press_to_start(master_bot) + + ### Teleoperation loop + gripper_command = JointSingleCommand(name="gripper") + while True: + # sync joint positions + master_state_joints = master_bot.dxl.joint_states.position[:6] + puppet_bot.arm.set_joint_positions(master_state_joints, blocking=False) + # sync gripper positions + master_gripper_joint = master_bot.dxl.joint_states.position[6] + puppet_gripper_joint_target = MASTER2PUPPET_JOINT_FN(master_gripper_joint) + gripper_command.cmd = puppet_gripper_joint_target + puppet_bot.gripper.core.pub_single.publish(gripper_command) + # sleep DT + time.sleep(DT) + + +if __name__=='__main__': + side = sys.argv[1] + teleop(side) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/real_env.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/real_env.py new file mode 100644 index 00000000..ded190c0 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/real_env.py @@ -0,0 +1,205 @@ +import time +import numpy as np +import collections +import matplotlib.pyplot as plt +import dm_env + +from lerobot_constants import DT, START_ARM_POSE, MASTER_GRIPPER_JOINT_NORMALIZE_FN, PUPPET_GRIPPER_JOINT_UNNORMALIZE_FN +from lerobot_constants import PUPPET_GRIPPER_POSITION_NORMALIZE_FN, PUPPET_GRIPPER_VELOCITY_NORMALIZE_FN +from lerobot_constants import PUPPET_GRIPPER_JOINT_OPEN, PUPPET_GRIPPER_JOINT_CLOSE +from robot_utils import Recorder, ImageRecorder +from robot_utils import setup_master_bot, setup_puppet_bot, move_arms, move_grippers +from interbotix_xs_modules.arm import InterbotixManipulatorXS +from interbotix_xs_msgs.msg import JointSingleCommand + +import IPython +e = IPython.embed + +class RealEnv: + """ + Environment for real robot bi-manual manipulation + Action space: [left_arm_qpos (6), # absolute joint position + left_gripper_positions (1), # normalized gripper position (0: close, 1: open) + right_arm_qpos (6), # absolute joint position + right_gripper_positions (1),] # normalized gripper position (0: close, 1: open) + + Observation space: {"qpos": Concat[ left_arm_qpos (6), # absolute joint position + left_gripper_position (1), # normalized gripper position (0: close, 1: open) + right_arm_qpos (6), # absolute joint position + right_gripper_qpos (1)] # normalized gripper position (0: close, 1: open) + "qvel": Concat[ left_arm_qvel (6), # absolute joint velocity (rad) + left_gripper_velocity (1), # normalized gripper velocity (pos: opening, neg: closing) + right_arm_qvel (6), # absolute joint velocity (rad) + right_gripper_qvel (1)] # normalized gripper velocity (pos: opening, neg: closing) + "images": {"cam_high": (480x640x3), # h, w, c, dtype='uint8' + "cam_low": (480x640x3), # h, w, c, dtype='uint8' + "cam_left_wrist": (480x640x3), # h, w, c, dtype='uint8' + "cam_right_wrist": (480x640x3)} # h, w, c, dtype='uint8' + """ + + def __init__(self, init_node, setup_robots=True): + self.puppet_bot_left = InterbotixManipulatorXS(robot_model="vx300s", group_name="arm", gripper_name="gripper", + robot_name=f'puppet_left', init_node=init_node) + self.puppet_bot_right = InterbotixManipulatorXS(robot_model="vx300s", group_name="arm", gripper_name="gripper", + robot_name=f'puppet_right', init_node=False) + if setup_robots: + self.setup_robots() + + self.recorder_left = Recorder('left', init_node=False) + self.recorder_right = Recorder('right', init_node=False) + self.image_recorder = ImageRecorder(init_node=False) + self.gripper_command = JointSingleCommand(name="gripper") + + def setup_robots(self): + setup_puppet_bot(self.puppet_bot_left) + setup_puppet_bot(self.puppet_bot_right) + + def get_qpos(self): + left_qpos_raw = self.recorder_left.qpos + right_qpos_raw = self.recorder_right.qpos + left_arm_qpos = left_qpos_raw[:6] + right_arm_qpos = right_qpos_raw[:6] + left_gripper_qpos = [PUPPET_GRIPPER_POSITION_NORMALIZE_FN(left_qpos_raw[7])] # this is position not joint + right_gripper_qpos = [PUPPET_GRIPPER_POSITION_NORMALIZE_FN(right_qpos_raw[7])] # this is position not joint + return np.concatenate([left_arm_qpos, left_gripper_qpos, right_arm_qpos, right_gripper_qpos]) + + def get_qvel(self): + left_qvel_raw = self.recorder_left.qvel + right_qvel_raw = self.recorder_right.qvel + left_arm_qvel = left_qvel_raw[:6] + right_arm_qvel = right_qvel_raw[:6] + left_gripper_qvel = [PUPPET_GRIPPER_VELOCITY_NORMALIZE_FN(left_qvel_raw[7])] + right_gripper_qvel = [PUPPET_GRIPPER_VELOCITY_NORMALIZE_FN(right_qvel_raw[7])] + return np.concatenate([left_arm_qvel, left_gripper_qvel, right_arm_qvel, right_gripper_qvel]) + + def get_effort(self): + left_effort_raw = self.recorder_left.effort + right_effort_raw = self.recorder_right.effort + left_robot_effort = left_effort_raw[:7] + right_robot_effort = right_effort_raw[:7] + return np.concatenate([left_robot_effort, right_robot_effort]) + + def get_images(self): + return self.image_recorder.get_images() + + def set_gripper_pose(self, left_gripper_desired_pos_normalized, right_gripper_desired_pos_normalized): + left_gripper_desired_joint = PUPPET_GRIPPER_JOINT_UNNORMALIZE_FN(left_gripper_desired_pos_normalized) + self.gripper_command.cmd = left_gripper_desired_joint + self.puppet_bot_left.gripper.core.pub_single.publish(self.gripper_command) + + right_gripper_desired_joint = PUPPET_GRIPPER_JOINT_UNNORMALIZE_FN(right_gripper_desired_pos_normalized) + self.gripper_command.cmd = right_gripper_desired_joint + self.puppet_bot_right.gripper.core.pub_single.publish(self.gripper_command) + + def _reset_joints(self): + reset_position = START_ARM_POSE[:6] + move_arms([self.puppet_bot_left, self.puppet_bot_right], [reset_position, reset_position], move_time=1) + + def _reset_gripper(self): + """Set to position mode and do position resets: first open then close. Then change back to PWM mode""" + move_grippers([self.puppet_bot_left, self.puppet_bot_right], [PUPPET_GRIPPER_JOINT_OPEN] * 2, move_time=0.5) + move_grippers([self.puppet_bot_left, self.puppet_bot_right], [PUPPET_GRIPPER_JOINT_CLOSE] * 2, move_time=1) + + def get_observation(self): + obs = collections.OrderedDict() + obs['qpos'] = self.get_qpos() + obs['qvel'] = self.get_qvel() + obs['effort'] = self.get_effort() + obs['images'] = self.get_images() + return obs + + def get_reward(self): + return 0 + + def reset(self, fake=False): + if not fake: + # Reboot puppet robot gripper motors + self.puppet_bot_left.dxl.robot_reboot_motors("single", "gripper", True) + self.puppet_bot_right.dxl.robot_reboot_motors("single", "gripper", True) + self._reset_joints() + self._reset_gripper() + return dm_env.TimeStep( + step_type=dm_env.StepType.FIRST, + reward=self.get_reward(), + discount=None, + observation=self.get_observation()) + + def step(self, action): + state_len = int(len(action) / 2) + left_action = action[:state_len] + right_action = action[state_len:] + self.puppet_bot_left.arm.set_joint_positions(left_action[:6], blocking=False) + self.puppet_bot_right.arm.set_joint_positions(right_action[:6], blocking=False) + self.set_gripper_pose(left_action[-1], right_action[-1]) + time.sleep(DT) + return dm_env.TimeStep( + step_type=dm_env.StepType.MID, + reward=self.get_reward(), + discount=None, + observation=self.get_observation()) + + +def get_action(master_bot_left, master_bot_right): + action = np.zeros(14) # 6 joint + 1 gripper, for two arms + # Arm actions + action[:6] = master_bot_left.dxl.joint_states.position[:6] + action[7:7+6] = master_bot_right.dxl.joint_states.position[:6] + # Gripper actions + action[6] = MASTER_GRIPPER_JOINT_NORMALIZE_FN(master_bot_left.dxl.joint_states.position[6]) + action[7+6] = MASTER_GRIPPER_JOINT_NORMALIZE_FN(master_bot_right.dxl.joint_states.position[6]) + + return action + + +def make_real_env(init_node, setup_robots=True): + env = RealEnv(init_node, setup_robots) + return env + + +def test_real_teleop(): + """ + Test bimanual teleoperation and show image observations onscreen. + It first reads joint poses from both master arms. + Then use it as actions to step the environment. + The environment returns full observations including images. + + An alternative approach is to have separate scripts for teleoperation and observation recording. + This script will result in higher fidelity (obs, action) pairs + """ + + onscreen_render = True + render_cam = 'cam_left_wrist' + + # source of data + master_bot_left = InterbotixManipulatorXS(robot_model="wx250s", group_name="arm", gripper_name="gripper", + robot_name=f'master_left', init_node=True) + master_bot_right = InterbotixManipulatorXS(robot_model="wx250s", group_name="arm", gripper_name="gripper", + robot_name=f'master_right', init_node=False) + setup_master_bot(master_bot_left) + setup_master_bot(master_bot_right) + + # setup the environment + env = make_real_env(init_node=False) + ts = env.reset(fake=True) + episode = [ts] + # setup visualization + if onscreen_render: + ax = plt.subplot() + plt_img = ax.imshow(ts.observation['images'][render_cam]) + plt.ion() + + for t in range(1000): + action = get_action(master_bot_left, master_bot_right) + ts = env.step(action) + episode.append(ts) + + if onscreen_render: + plt_img.set_data(ts.observation['images'][render_cam]) + plt.pause(DT) + else: + time.sleep(DT) + + +if __name__ == '__main__': + test_real_teleop() + diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/reasonings_constants.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/reasonings_constants.py new file mode 100644 index 00000000..31039a8e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/reasonings_constants.py @@ -0,0 +1,7 @@ +TASK_REASONINGS = { + 'fold_shirt': 'The t-shirt is flatten, fold it.', +} + +TASK_INSTRUCTIONS = { + 'fold_shirt': 'Fold t-shirt on the table.', +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/record_episodes.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/record_episodes.py new file mode 100644 index 00000000..34f0e54a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/record_episodes.py @@ -0,0 +1,228 @@ +import os +import time +import h5py +import argparse +import numpy as np +from tqdm import tqdm + +from lerobot_constants import DT, START_ARM_POSE, TASK_CONFIGS +from lerobot_constants import MASTER_GRIPPER_JOINT_MID, PUPPET_GRIPPER_JOINT_CLOSE, PUPPET_GRIPPER_JOINT_OPEN +from robot_utils import Recorder, ImageRecorder, get_arm_gripper_positions +from robot_utils import move_arms, torque_on, torque_off, move_grippers +from real_env import make_real_env, get_action + +from interbotix_xs_modules.arm import InterbotixManipulatorXS + +import IPython +e = IPython.embed + + +def opening_ceremony(master_bot_left, master_bot_right, puppet_bot_left, puppet_bot_right): + """ Move all 4 robots to a pose where it is easy to start demonstration """ + # reboot gripper motors, and set operating modes for all motors + puppet_bot_left.dxl.robot_reboot_motors("single", "gripper", True) + puppet_bot_left.dxl.robot_set_operating_modes("group", "arm", "position") + puppet_bot_left.dxl.robot_set_operating_modes("single", "gripper", "current_based_position") + master_bot_left.dxl.robot_set_operating_modes("group", "arm", "position") + master_bot_left.dxl.robot_set_operating_modes("single", "gripper", "position") + # puppet_bot_left.dxl.robot_set_motor_registers("single", "gripper", 'current_limit', 1000) # TODO(tonyzhaozh) figure out how to set this limit + + puppet_bot_right.dxl.robot_reboot_motors("single", "gripper", True) + puppet_bot_right.dxl.robot_set_operating_modes("group", "arm", "position") + puppet_bot_right.dxl.robot_set_operating_modes("single", "gripper", "current_based_position") + master_bot_right.dxl.robot_set_operating_modes("group", "arm", "position") + master_bot_right.dxl.robot_set_operating_modes("single", "gripper", "position") + # puppet_bot_left.dxl.robot_set_motor_registers("single", "gripper", 'current_limit', 1000) # TODO(tonyzhaozh) figure out how to set this limit + + torque_on(puppet_bot_left) + torque_on(master_bot_left) + torque_on(puppet_bot_right) + torque_on(master_bot_right) + + # move arms to starting position + start_arm_qpos = START_ARM_POSE[:6] + move_arms([master_bot_left, puppet_bot_left, master_bot_right, puppet_bot_right], [start_arm_qpos] * 4, move_time=1.5) + # move grippers to starting position + move_grippers([master_bot_left, puppet_bot_left, master_bot_right, puppet_bot_right], [MASTER_GRIPPER_JOINT_MID, PUPPET_GRIPPER_JOINT_CLOSE] * 2, move_time=0.5) + + + # press gripper to start data collection + # disable torque for only gripper joint of master robot to allow user movement + master_bot_left.dxl.robot_torque_enable("single", "gripper", False) + master_bot_right.dxl.robot_torque_enable("single", "gripper", False) + print(f'Close the gripper to start') + close_thresh = -0.3 + pressed = False + while not pressed: + gripper_pos_left = get_arm_gripper_positions(master_bot_left) + gripper_pos_right = get_arm_gripper_positions(master_bot_right) + if (gripper_pos_left < close_thresh) and (gripper_pos_right < close_thresh): + pressed = True + time.sleep(DT/10) + torque_off(master_bot_left) + torque_off(master_bot_right) + print(f'Started!') + + +def capture_one_episode(dt, max_timesteps, camera_names, dataset_dir, dataset_name, overwrite): + print(f'Dataset name: {dataset_name}') + + # source of data + master_bot_left = InterbotixManipulatorXS(robot_model="wx250s", group_name="arm", gripper_name="gripper", + robot_name=f'master_left', init_node=True) + master_bot_right = InterbotixManipulatorXS(robot_model="wx250s", group_name="arm", gripper_name="gripper", + robot_name=f'master_right', init_node=False) + env = make_real_env(init_node=False, setup_robots=False) + + # saving dataset + if not os.path.isdir(dataset_dir): + os.makedirs(dataset_dir) + dataset_path = os.path.join(dataset_dir, dataset_name) + if os.path.isfile(dataset_path) and not overwrite: + print(f'Dataset already exist at \n{dataset_path}\nHint: set overwrite to True.') + exit() + + # move all 4 robots to a starting pose where it is easy to start teleoperation, then wait till both gripper closed + opening_ceremony(master_bot_left, master_bot_right, env.puppet_bot_left, env.puppet_bot_right) + + # Data collection + ts = env.reset(fake=True) + timesteps = [ts] + actions = [] + actual_dt_history = [] + for t in tqdm(range(max_timesteps)): + t0 = time.time() # + action = get_action(master_bot_left, master_bot_right) + t1 = time.time() # + ts = env.step(action) + t2 = time.time() # + timesteps.append(ts) + actions.append(action) + actual_dt_history.append([t0, t1, t2]) + + # Torque on both master bots + torque_on(master_bot_left) + torque_on(master_bot_right) + # Open puppet grippers + move_grippers([env.puppet_bot_left, env.puppet_bot_right], [PUPPET_GRIPPER_JOINT_OPEN] * 2, move_time=0.5) + + freq_mean = print_dt_diagnosis(actual_dt_history) + if freq_mean < 42: + return False + + """ + For each timestep: + observations + - images + - cam_high (480, 640, 3) 'uint8' + - cam_low (480, 640, 3) 'uint8' + - cam_left_wrist (480, 640, 3) 'uint8' + - cam_right_wrist (480, 640, 3) 'uint8' + - qpos (14,) 'float64' + - qvel (14,) 'float64' + + action (14,) 'float64' + """ + + data_dict = { + '/observations/qpos': [], + '/observations/qvel': [], + '/observations/effort': [], + '/action': [], + } + for cam_name in camera_names: + data_dict[f'/observations/images/{cam_name}'] = [] + + # len(action): max_timesteps, len(time_steps): max_timesteps + 1 + while actions: + action = actions.pop(0) + ts = timesteps.pop(0) + data_dict['/observations/qpos'].append(ts.observation['qpos']) + data_dict['/observations/qvel'].append(ts.observation['qvel']) + data_dict['/observations/effort'].append(ts.observation['effort']) + data_dict['/action'].append(action) + for cam_name in camera_names: + data_dict[f'/observations/images/{cam_name}'].append(ts.observation['images'][cam_name]) + + # HDF5 + t0 = time.time() + with h5py.File(dataset_path + '.hdf5', 'w', rdcc_nbytes=1024**2*2) as root: + root.attrs['sim'] = False + obs = root.create_group('observations') + image = obs.create_group('images') + for cam_name in camera_names: + _ = image.create_dataset(cam_name, (max_timesteps, 480, 640, 3), dtype='uint8', + chunks=(1, 480, 640, 3), ) + # compression='gzip',compression_opts=2,) + # compression=32001, compression_opts=(0, 0, 0, 0, 9, 1, 1), shuffle=False) + _ = obs.create_dataset('qpos', (max_timesteps, 14)) + _ = obs.create_dataset('qvel', (max_timesteps, 14)) + _ = obs.create_dataset('effort', (max_timesteps, 14)) + _ = root.create_dataset('action', (max_timesteps, 14)) + + for name, array in data_dict.items(): + root[name][...] = array + print(f'Saving: {time.time() - t0:.1f} secs') + + return True + + +def main(args): + task_config = TASK_CONFIGS[args['task_name']] + dataset_dir = task_config['dataset_dir'] + max_timesteps = task_config['episode_len'] + camera_names = task_config['camera_names'] + + if args['episode_idx'] is not None: + episode_idx = args['episode_idx'] + else: + episode_idx = get_auto_index(dataset_dir) + overwrite = True + + dataset_name = f'episode_{episode_idx}' + print(dataset_name + '\n') + while True: + is_healthy = capture_one_episode(DT, max_timesteps, camera_names, dataset_dir, dataset_name, overwrite) + if is_healthy: + break + + +def get_auto_index(dataset_dir, dataset_name_prefix = '', data_suffix = 'hdf5'): + max_idx = 1000 + if not os.path.isdir(dataset_dir): + os.makedirs(dataset_dir) + for i in range(max_idx+1): + if not os.path.isfile(os.path.join(dataset_dir, f'{dataset_name_prefix}episode_{i}.{data_suffix}')): + return i + raise Exception(f"Error getting auto index, or more than {max_idx} episodes") + + +def print_dt_diagnosis(actual_dt_history): + actual_dt_history = np.array(actual_dt_history) + get_action_time = actual_dt_history[:, 1] - actual_dt_history[:, 0] + step_env_time = actual_dt_history[:, 2] - actual_dt_history[:, 1] + total_time = actual_dt_history[:, 2] - actual_dt_history[:, 0] + + dt_mean = np.mean(total_time) + dt_std = np.std(total_time) + freq_mean = 1 / dt_mean + print(f'Avg freq: {freq_mean:.2f} Get action: {np.mean(get_action_time):.3f} Step env: {np.mean(step_env_time):.3f}') + return freq_mean + +def debug(): + print(f'====== Debug mode ======') + recorder = Recorder('right', is_debug=True) + image_recorder = ImageRecorder(init_node=False, is_debug=True) + while True: + time.sleep(1) + recorder.print_diagnostics() + image_recorder.print_diagnostics() + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--task_name', action='store', type=str, help='Task name.', required=True) + parser.add_argument('--episode_idx', action='store', type=int, help='Episode index.', default=None, required=False) + main(vars(parser.parse_args())) + # debug() + + diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/replay_episodes.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/replay_episodes.py new file mode 100644 index 00000000..c5b017e8 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/replay_episodes.py @@ -0,0 +1,40 @@ +import os +import h5py +from robot_utils import move_grippers +import argparse +from real_env import make_real_env +from lerobot_constants import JOINT_NAMES, PUPPET_GRIPPER_JOINT_OPEN + +import IPython +e = IPython.embed + +STATE_NAMES = JOINT_NAMES + ["gripper", 'left_finger', 'right_finger'] + +def main(args): + dataset_dir = args['dataset_dir'] + episode_idx = args['episode_idx'] + dataset_name = f'episode_{episode_idx}' + + dataset_path = os.path.join(dataset_dir, dataset_name + '.hdf5') + if not os.path.isfile(dataset_path): + print(f'Dataset does not exist at \n{dataset_path}\n') + exit() + + with h5py.File(dataset_path, 'r') as root: + actions = root['/action'][()] + + env = make_real_env(init_node=True) + env.reset() + for action in actions: + env.step(action) + + move_grippers([env.puppet_bot_left, env.puppet_bot_right], [PUPPET_GRIPPER_JOINT_OPEN] * 2, move_time=0.5) # open + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--dataset_dir', action='store', type=str, help='Dataset dir.', required=True) + parser.add_argument('--episode_idx', action='store', type=int, help='Episode index.', required=False) + main(vars(parser.parse_args())) + + diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/robot_utils.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/robot_utils.py new file mode 100644 index 00000000..83908742 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/robot_utils.py @@ -0,0 +1,187 @@ +import numpy as np +import time +from lerobot_constants import DT +from interbotix_xs_msgs.msg import JointSingleCommand + +import IPython +e = IPython.embed + +class ImageRecorder: + def __init__(self, init_node=True, is_debug=False): + from collections import deque + import rospy + from cv_bridge import CvBridge + from sensor_msgs.msg import Image + self.is_debug = is_debug + self.bridge = CvBridge() + self.camera_names = ['cam_high', 'cam_low', 'cam_left_wrist', 'cam_right_wrist'] + if init_node: + rospy.init_node('image_recorder', anonymous=True) + for cam_name in self.camera_names: + setattr(self, f'{cam_name}_image', None) + setattr(self, f'{cam_name}_secs', None) + setattr(self, f'{cam_name}_nsecs', None) + if cam_name == 'cam_high': + callback_func = self.image_cb_cam_high + elif cam_name == 'cam_low': + callback_func = self.image_cb_cam_low + elif cam_name == 'cam_left_wrist': + callback_func = self.image_cb_cam_left_wrist + elif cam_name == 'cam_right_wrist': + callback_func = self.image_cb_cam_right_wrist + else: + raise NotImplementedError + rospy.Subscriber(f"/usb_{cam_name}/image_raw", Image, callback_func) + if self.is_debug: + setattr(self, f'{cam_name}_timestamps', deque(maxlen=50)) + time.sleep(0.5) + + def image_cb(self, cam_name, data): + setattr(self, f'{cam_name}_image', self.bridge.imgmsg_to_cv2(data, desired_encoding='passthrough')) + setattr(self, f'{cam_name}_secs', data.header.stamp.secs) + setattr(self, f'{cam_name}_nsecs', data.header.stamp.nsecs) + # cv2.imwrite('/home/tonyzhao/Desktop/sample.jpg', cv_image) + if self.is_debug: + getattr(self, f'{cam_name}_timestamps').append(data.header.stamp.secs + data.header.stamp.secs * 1e-9) + + def image_cb_cam_high(self, data): + cam_name = 'cam_high' + return self.image_cb(cam_name, data) + + def image_cb_cam_low(self, data): + cam_name = 'cam_low' + return self.image_cb(cam_name, data) + + def image_cb_cam_left_wrist(self, data): + cam_name = 'cam_left_wrist' + return self.image_cb(cam_name, data) + + def image_cb_cam_right_wrist(self, data): + cam_name = 'cam_right_wrist' + return self.image_cb(cam_name, data) + + def get_images(self): + image_dict = dict() + for cam_name in self.camera_names: + image_dict[cam_name] = getattr(self, f'{cam_name}_image') + return image_dict + + def print_diagnostics(self): + def dt_helper(l): + l = np.array(l) + diff = l[1:] - l[:-1] + return np.mean(diff) + for cam_name in self.camera_names: + image_freq = 1 / dt_helper(getattr(self, f'{cam_name}_timestamps')) + print(f'{cam_name} {image_freq=:.2f}') + print() + +class Recorder: + def __init__(self, side, init_node=True, is_debug=False): + from collections import deque + import rospy + from sensor_msgs.msg import JointState + from interbotix_xs_msgs.msg import JointGroupCommand, JointSingleCommand + + self.secs = None + self.nsecs = None + self.qpos = None + self.effort = None + self.arm_command = None + self.gripper_command = None + self.is_debug = is_debug + + if init_node: + rospy.init_node('recorder', anonymous=True) + rospy.Subscriber(f"/puppet_{side}/joint_states", JointState, self.puppet_state_cb) + rospy.Subscriber(f"/puppet_{side}/commands/joint_group", JointGroupCommand, self.puppet_arm_commands_cb) + rospy.Subscriber(f"/puppet_{side}/commands/joint_single", JointSingleCommand, self.puppet_gripper_commands_cb) + if self.is_debug: + self.joint_timestamps = deque(maxlen=50) + self.arm_command_timestamps = deque(maxlen=50) + self.gripper_command_timestamps = deque(maxlen=50) + time.sleep(0.1) + + def puppet_state_cb(self, data): + self.qpos = data.position + self.qvel = data.velocity + self.effort = data.effort + self.data = data + if self.is_debug: + self.joint_timestamps.append(time.time()) + + def puppet_arm_commands_cb(self, data): + self.arm_command = data.cmd + if self.is_debug: + self.arm_command_timestamps.append(time.time()) + + def puppet_gripper_commands_cb(self, data): + self.gripper_command = data.cmd + if self.is_debug: + self.gripper_command_timestamps.append(time.time()) + + def print_diagnostics(self): + def dt_helper(l): + l = np.array(l) + diff = l[1:] - l[:-1] + return np.mean(diff) + + joint_freq = 1 / dt_helper(self.joint_timestamps) + arm_command_freq = 1 / dt_helper(self.arm_command_timestamps) + gripper_command_freq = 1 / dt_helper(self.gripper_command_timestamps) + + print(f'{joint_freq=:.2f}\n{arm_command_freq=:.2f}\n{gripper_command_freq=:.2f}\n') + +def get_arm_joint_positions(bot): + return bot.arm.core.joint_states.position[:6] + +def get_arm_gripper_positions(bot): + joint_position = bot.gripper.core.joint_states.position[6] + return joint_position + +def move_arms(bot_list, target_pose_list, move_time=1): + num_steps = int(move_time / DT) + curr_pose_list = [get_arm_joint_positions(bot) for bot in bot_list] + traj_list = [np.linspace(curr_pose, target_pose, num_steps) for curr_pose, target_pose in zip(curr_pose_list, target_pose_list)] + for t in range(num_steps): + for bot_id, bot in enumerate(bot_list): + bot.arm.set_joint_positions(traj_list[bot_id][t], blocking=False) + time.sleep(DT) + +def move_grippers(bot_list, target_pose_list, move_time): + gripper_command = JointSingleCommand(name="gripper") + num_steps = int(move_time / DT) + curr_pose_list = [get_arm_gripper_positions(bot) for bot in bot_list] + traj_list = [np.linspace(curr_pose, target_pose, num_steps) for curr_pose, target_pose in zip(curr_pose_list, target_pose_list)] + for t in range(num_steps): + for bot_id, bot in enumerate(bot_list): + gripper_command.cmd = traj_list[bot_id][t] + bot.gripper.core.pub_single.publish(gripper_command) + time.sleep(DT) + +def setup_puppet_bot(bot): + bot.dxl.robot_reboot_motors("single", "gripper", True) + bot.dxl.robot_set_operating_modes("group", "arm", "position") + bot.dxl.robot_set_operating_modes("single", "gripper", "current_based_position") + torque_on(bot) + +def setup_master_bot(bot): + bot.dxl.robot_set_operating_modes("group", "arm", "pwm") + bot.dxl.robot_set_operating_modes("single", "gripper", "current_based_position") + torque_off(bot) + +def set_standard_pid_gains(bot): + bot.dxl.robot_set_motor_registers("group", "arm", 'Position_P_Gain', 800) + bot.dxl.robot_set_motor_registers("group", "arm", 'Position_I_Gain', 0) + +def set_low_pid_gains(bot): + bot.dxl.robot_set_motor_registers("group", "arm", 'Position_P_Gain', 100) + bot.dxl.robot_set_motor_registers("group", "arm", 'Position_I_Gain', 0) + +def torque_off(bot): + bot.dxl.robot_torque_enable("group", "arm", False) + bot.dxl.robot_torque_enable("single", "gripper", False) + +def torque_on(bot): + bot.dxl.robot_torque_enable("group", "arm", True) + bot.dxl.robot_torque_enable("single", "gripper", True) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/sleep.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/sleep.py new file mode 100644 index 00000000..3567fa55 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/sleep.py @@ -0,0 +1,19 @@ +from interbotix_xs_modules.arm import InterbotixManipulatorXS +from robot_utils import move_arms, torque_on + +def main(): + puppet_bot_left = InterbotixManipulatorXS(robot_model="vx300s", group_name="arm", gripper_name="gripper", robot_name=f'puppet_left', init_node=True) + puppet_bot_right = InterbotixManipulatorXS(robot_model="vx300s", group_name="arm", gripper_name="gripper", robot_name=f'puppet_right', init_node=False) + master_bot_left = InterbotixManipulatorXS(robot_model="wx250s", group_name="arm", gripper_name="gripper", robot_name=f'master_left', init_node=False) + master_bot_right = InterbotixManipulatorXS(robot_model="wx250s", group_name="arm", gripper_name="gripper", robot_name=f'master_right', init_node=False) + + all_bots = [puppet_bot_left, puppet_bot_right] + for bot in all_bots: + torque_on(bot) + + puppet_sleep_position = (0, -1.7, 1.55, 0.12, 0.65, 0) + master_sleep_position = (0, -1.1, 1.24, 0, -0.24, 0) + move_arms(all_bots, [puppet_sleep_position] * 2, move_time=2) + +if __name__ == '__main__': + main() diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/utils.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/utils.py new file mode 100644 index 00000000..25f3e947 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/utils.py @@ -0,0 +1,5 @@ +RED = '\033[31m' +GREEN = '\033[32m' +YELLOW = '\033[33m' +BLUE = '\033[34m' +RESET = '\033[0m' # Reset to default color \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/visualize_episodes.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/visualize_episodes.py new file mode 100644 index 00000000..a96d3bbe --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/aloha_scripts/visualize_episodes.py @@ -0,0 +1,187 @@ +import os +import numpy as np +import cv2 +import h5py +import argparse + +import matplotlib.pyplot as plt +from PIL import Image +import IPython +from tqdm import tqdm +e = IPython.embed + +JOINT_NAMES = ["waist", "shoulder", "elbow", "forearm_roll", "wrist_angle", "wrist_rotate"] +STATE_NAMES = JOINT_NAMES + ["gripper"] + +def load_hdf5(dataset_dir, dataset_name): + dataset_path = os.path.join(dataset_dir, dataset_name + '.hdf5') + if not os.path.isfile(dataset_path): + print(f'Dataset does not exist at \n{dataset_path}\n') + exit() + + with h5py.File(dataset_path, 'r') as root: + is_sim = root.attrs['sim'] + qpos = root['/observations/qpos'][()] + qvel = root['/observations/qvel'][()] + effort = root['/observations/effort'][()] + action = root['/action'][()] + image_dict = dict() + for cam_name in root[f'/observations/images/'].keys(): + image_dict[cam_name] = root[f'/observations/images/{cam_name}'][()] + + return qpos, qvel, effort, action, image_dict + +def main(args): + dataset_dir = args['dataset_dir'] + episode_idx = args['episode_idx'] + dataset_name = f'episode_{episode_idx}' + + qpos, qvel, effort, action, image_dict = load_hdf5(dataset_dir, dataset_name) + save_images(image_dict, image_path=os.path.join(dataset_dir, dataset_name)) + # save_videos(image_dict, DT, video_path=os.path.join(dataset_dir, dataset_name + '_video.mp4')) + visualize_joints(qpos, action, plot_path=os.path.join(dataset_dir, dataset_name + '_qpos.png')) + visualize_single(effort, 'effort', plot_path=os.path.join(dataset_dir, dataset_name + '_effort.png')) + visualize_single(action - qpos, 'tracking_error', plot_path=os.path.join(dataset_dir, dataset_name + '_error.png')) + # visualize_timestamp(t_list, dataset_path) # TODO addn timestamp back + + +def save_videos(video, dt, video_path=None): + if isinstance(video, list): + cam_names = list(video[0].keys()) + h, w, _ = video[0][cam_names[0]].shape + w = w * len(cam_names) + fps = int(1/dt) + out = cv2.VideoWriter(video_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h)) + for ts, image_dict in enumerate(video): + images = [] + for cam_name in cam_names: + image = image_dict[cam_name] + image = image[:, :, [2, 1, 0]] # swap B and R channel + images.append(image) + images = np.concatenate(images, axis=1) + out.write(images) + out.release() + print(f'Saved video to: {video_path}') + elif isinstance(video, dict): + cam_names = list(video.keys()) + all_cam_videos = [] + for cam_name in cam_names: + all_cam_videos.append(video[cam_name]) + all_cam_videos = np.concatenate(all_cam_videos, axis=2) # width dimension + + n_frames, h, w, _ = all_cam_videos.shape + fps = int(1 / dt) + out = cv2.VideoWriter(video_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h)) + for t in range(n_frames): + image = all_cam_videos[t] + image = image[:, :, [2, 1, 0]] # swap B and R channel + out.write(image) + out.release() + print(f'Saved video to: {video_path}') + +def save_images(video, image_path=None): + cam_names = list(video.keys()) + for cam_name in cam_names: + cam_path = os.path.join(image_path, cam_name) + os.makedirs(cam_path, exist_ok=True) + for idx, img in tqdm(enumerate(video[cam_name])): + pil = Image.fromarray(img) + pil.save(os.path.join(cam_path, f"{idx}.png")) + + print(f'Saved images to: {image_path}') + +def visualize_joints(qpos_list, command_list, plot_path=None, ylim=None, label_overwrite=None): + if label_overwrite: + label1, label2 = label_overwrite + else: + label1, label2 = 'State', 'Command' + + qpos = np.array(qpos_list) # ts, dim + command = np.array(command_list) + num_ts, num_dim = qpos.shape + h, w = 2, num_dim + num_figs = num_dim + fig, axs = plt.subplots(num_figs, 1, figsize=(w, h * num_figs)) + + # plot joint state + all_names = [name + '_left' for name in STATE_NAMES] + [name + '_right' for name in STATE_NAMES] + for dim_idx in range(num_dim): + ax = axs[dim_idx] + ax.plot(qpos[:, dim_idx], label=label1) + ax.set_title(f'Joint {dim_idx}: {all_names[dim_idx]}') + ax.legend() + + # plot arm command + for dim_idx in range(num_dim): + ax = axs[dim_idx] + ax.plot(command[:, dim_idx], label=label2) + ax.legend() + + if ylim: + for dim_idx in range(num_dim): + ax = axs[dim_idx] + ax.set_ylim(ylim) + + plt.tight_layout() + plt.savefig(plot_path) + print(f'Saved qpos plot to: {plot_path}') + plt.close() + +def visualize_single(efforts_list, label, plot_path=None, ylim=None, label_overwrite=None): + efforts = np.array(efforts_list) # ts, dim + num_ts, num_dim = efforts.shape + h, w = 2, num_dim + num_figs = num_dim + fig, axs = plt.subplots(num_figs, 1, figsize=(w, h * num_figs)) + + # plot joint state + all_names = [name + '_left' for name in STATE_NAMES] + [name + '_right' for name in STATE_NAMES] + for dim_idx in range(num_dim): + ax = axs[dim_idx] + ax.plot(efforts[:, dim_idx], label=label) + ax.set_title(f'Joint {dim_idx}: {all_names[dim_idx]}') + ax.legend() + + if ylim: + for dim_idx in range(num_dim): + ax = axs[dim_idx] + ax.set_ylim(ylim) + + plt.tight_layout() + plt.savefig(plot_path) + print(f'Saved effort plot to: {plot_path}') + plt.close() + + +def visualize_timestamp(t_list, dataset_path): + plot_path = dataset_path.replace('.pkl', '_timestamp.png') + h, w = 4, 10 + fig, axs = plt.subplots(2, 1, figsize=(w, h*2)) + # process t_list + t_float = [] + for secs, nsecs in t_list: + t_float.append(secs + nsecs * 10E-10) + t_float = np.array(t_float) + + ax = axs[0] + ax.plot(np.arange(len(t_float)), t_float) + ax.set_title(f'Camera frame timestamps') + ax.set_xlabel('timestep') + ax.set_ylabel('time (sec)') + + ax = axs[1] + ax.plot(np.arange(len(t_float)-1), t_float[:-1] - t_float[1:]) + ax.set_title(f'dt') + ax.set_xlabel('timestep') + ax.set_ylabel('time (sec)') + + plt.tight_layout() + plt.savefig(plot_path) + print(f'Saved timestamp plot to: {plot_path}') + plt.close() + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--dataset_dir', default="/media/rl/HDD/data/data/droid_h5py/folding_shirt", type=str, help='Dataset dir.', required=False) + parser.add_argument('--episode_idx', default=0, type=int, help='Episode index.', required=False) + main(vars(parser.parse_args())) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/conda_env.yaml b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/conda_env.yaml new file mode 100644 index 00000000..8a436f71 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/conda_env.yaml @@ -0,0 +1,23 @@ +name: dexvla +channels: + - pytorch + - nvidia + - conda-forge +dependencies: + - python=3.9 + - pip=23.0.1 + - pytorch=2.0.0 + - torchvision=0.15.0 + - pytorch-cuda=11.8 + - pyquaternion=0.9.9 + - pyyaml=6.0 + - rospkg=1.5.0 + - pexpect=4.8.0 + - mujoco=2.3.3 + - dm_control=1.0.9 + - py-opencv=4.7.0 + - matplotlib=3.7.1 + - einops=0.6.0 + - packaging=23.0 + - h5py=3.8.0 + - ipython=8.12.0 diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/data_utils/.ipynb_checkpoints/dataset-checkpoint.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/data_utils/.ipynb_checkpoints/dataset-checkpoint.py new file mode 100644 index 00000000..7eb13429 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/data_utils/.ipynb_checkpoints/dataset-checkpoint.py @@ -0,0 +1,543 @@ +import numpy as np +import torch +import os +import h5py +import pickle +import fnmatch +import cv2 +from time import time +from torch.utils.data import TensorDataset, DataLoader +import torchvision.transforms as transforms +from torchvision.transforms.functional import to_pil_image, to_tensor +import IPython +import copy +e = IPython.embed +from aloha_scripts.utils import * + +def flatten_list(l): + return [item for sublist in l for item in sublist] +import gc +class EpisodicDataset(torch.utils.data.Dataset): + def __init__(self, dataset_path_list, camera_names, norm_stats, episode_ids, episode_len, chunk_size, policy_class, robot=None, rank0_print=print, llava_pythia_process=None, data_args=None, action_args=None): + super(EpisodicDataset).__init__() + self.episode_ids = episode_ids + self.dataset_path_list = dataset_path_list + self.camera_names = camera_names + self.norm_stats = norm_stats + self.episode_len = episode_len + self.chunk_size = chunk_size + self.cumulative_len = np.cumsum(self.episode_len) + self.max_episode_len = max(episode_len) + self.policy_class = policy_class + self.llava_pythia_process = llava_pythia_process + self.data_args = data_args + self.action_args = action_args + self.robot = robot + self.rank0_print = rank0_print + + original_size = (480, 640) + new_size = eval(self.data_args.image_size_stable) # 320, 240 + new_size = (new_size[1], new_size[0]) + ratio = 0.95 + self.transformations = [ + # todo resize + # transforms.Resize(size=original_size, antialias=True), + transforms.RandomCrop(size=[int(original_size[0] * ratio), int(original_size[1] * ratio)]), + transforms.Resize(original_size, antialias=True), + transforms.RandomRotation(degrees=[-5.0, 5.0], expand=False), + transforms.ColorJitter(brightness=0.3, contrast=0.4, saturation=0.5), # , hue=0.08) + transforms.Resize(size=new_size, antialias=True), + ] + + self.rank0_print(f"########################Current Image Size is [{self.data_args.image_size_stable}]###################################") + self.rank0_print(f"{RED}policy class: {self.policy_class}; augument: {True}{RESET}") + a=self.__getitem__(0) # initialize self.is_sim and self.transformations + if len(self.camera_names) > 2: + # self.rank0_print("%"*40) + self.rank0_print(f"The robot is {RED} {self.robot} {RESET} | The camera views: {RED} {self.camera_names} {RESET} | The history length: {RED} {self.data_args.history_images_length} {RESET}") + self.is_sim = False + + def __len__(self): + return sum(self.episode_len) + + def _locate_transition(self, index): + assert index < self.cumulative_len[-1] + episode_index = np.argmax(self.cumulative_len > index) # argmax returns first True index + start_ts = index - (self.cumulative_len[episode_index] - self.episode_len[episode_index]) + episode_id = self.episode_ids[episode_index] + return episode_id, start_ts + + def load_from_h5(self, dataset_path, start_ts): + + task_base_name = os.path.basename(dataset_path).replace('.hdf5', '') + task_dir_name = os.path.basename(os.path.dirname(dataset_path)) + annotation_path = os.path.join(os.path.dirname(dataset_path), task_dir_name + '.pkl') + + with open(annotation_path, 'rb') as f: + annotation_data = pickle.load(f) + + # get instruction + try: + raw_lang = annotation_data[task_base_name]['instruction'] + except Exception as e: + # self.rank0_print(e) + self.rank0_print(f"Read {dataset_path} happens {YELLOW}{e}{RESET}") + exit(0) + + # get sub reason of current frame + reasoning = "None." + if self.data_args.use_reasoning: + try: + reasoning = annotation_data[task_base_name]['sub_reason'][start_ts] + except Exception as e: + self.rank0_print(f"Read reasoning from {dataset_path} happens {YELLOW}{e}{RESET}") + exit(0) + + with h5py.File(dataset_path, 'r') as root: + + compressed = root.attrs.get('compress', False) + # try: + # raw_lang = root['language_raw'][0].decode('utf-8') + # except Exception as e: + # # self.rank0_print(e) + # self.rank0_print(f"Read {dataset_path} happens {YELLOW}{e}{RESET}") + # exit(0) + # reasoning = "None." + # if self.data_args.use_reasoning: + # if 'substep_reasonings' in root.keys(): + # reasoning = root['substep_reasonings'][start_ts].decode('utf-8') + # else: + # # print("no substep reasonings") + # try: + # reasoning = root['reasoning'][0].decode('utf-8') + # except Exception as e: + # # self.rank0_print(e) + # self.rank0_print(f"Read reasoning from {dataset_path} happens {YELLOW}{e}{RESET}") + # exit(0) + + try: # only used for agelix and franka + qpos = root['/observations/qpos'][start_ts] + action = root['/action'][()][:, :] + except: # for mobile aloha + if not root.get('/state/base_vel', None): + qpos = np.concatenate([ + root['/state/joint_position/left'][()][:-1], + root['/state/joint_position/right'][()][:-1], + # root['/state/base_vel'][()][:-1] + ], + axis=1)[start_ts] + action = np.concatenate([ + root['/state/joint_position/left'][()][1:], + root['/state/joint_position/right'][()][1:], + # root['/action/base_vel'][()][:-1] + ], + axis=1) + else: + qpos = np.concatenate([ + root['/state/joint_position/left'][()][:-1], + root['/state/joint_position/right'][()][:-1], + root['/state/base_vel'][()][:-1]], + axis=1)[start_ts] + action = np.concatenate([ + root['/state/joint_position/left'][()][1:], + root['/state/joint_position/right'][()][1:], + root['/action/base_vel'][()][:-1]], + axis=1) + + # print(f'======debug qpos load h5: {qpos.shape}') + # print(f'======debug action load h5: {action.shape}') + qpos = qpos[:self.action_args.action_dim] + action = action[:, :self.action_args.action_dim] + # print(f'======debug qpos load h5 aft: {qpos.shape}') + # print(f'======debug action load h5 aft: {action.shape}') + original_action_shape = action.shape + episode_len = original_action_shape[0] + image_dict = dict() + for cam_name in self.camera_names: + image_dict[cam_name] = root[f'/observations/images/{cam_name}'][start_ts] + + if compressed: + for cam_name in image_dict.keys(): + decompressed_image = cv2.imdecode(image_dict[cam_name], 1) + image_dict[cam_name] = np.array(decompressed_image) + + # get all actions after and including start_ts + # action = action[start_ts:] # hack, to make timesteps more aligned + # action_len = episode_len - start_ts # hack, to make timesteps more aligned + action = action[max(0, start_ts - 1):] # hack, to make timesteps more aligned + action_len = episode_len - max(0, start_ts - 1) # hack, to make timesteps more aligned + return original_action_shape, action, action_len, image_dict, qpos, raw_lang, reasoning + def __getitem__(self, index): + episode_id, start_ts = self._locate_transition(index) + dataset_path = self.dataset_path_list[episode_id] + # print(dataset_path) + try: + original_action_shape, action, action_len, image_dict, qpos, raw_lang, reasoning = self.load_from_h5(dataset_path, start_ts) + except Exception as e: + print(f"Read {dataset_path} happens {YELLOW}{e}{RESET}") + try: + dataset_path = self.dataset_path_list[episode_id + 1] + except Exception as e: + dataset_path = self.dataset_path_list[episode_id - 1] + + original_action_shape, action, action_len, image_dict, qpos, raw_lang, reasoning = self.load_from_h5(dataset_path, start_ts) + + # self.is_sim = is_sim + padded_action = np.zeros((self.max_episode_len, original_action_shape[1]), dtype=np.float32) + + padded_action[:action_len] = action + is_pad = np.zeros(self.max_episode_len) + is_pad[action_len:] = 1 + + padded_action = padded_action[:self.chunk_size] + is_pad = is_pad[:self.chunk_size] + + # new axis for different cameras + all_cam_images = [] + for cam_name in self.camera_names: + all_cam_images.append(image_dict[cam_name]) + all_cam_images = np.stack(all_cam_images, axis=0) + + # construct observations + image_data = torch.from_numpy(all_cam_images) + qpos_data = torch.from_numpy(qpos).float() + action_data = torch.from_numpy(padded_action).float() + is_pad = torch.from_numpy(is_pad).bool() + + # if 'top' in self.camera_names or 'cam_high' in self.camera_names: # denote for data collect via bimanual UR5 + if self.robot == 'franka': + assert image_data.ndim==4, f"image_data's shape is {image_data.shape}, maybe the reason of adding historical images" + image_data = torch.stack([torch.from_numpy(cv2.cvtColor(img.numpy(), cv2.COLOR_BGR2RGB)) for img in image_data], dim=0) + + # channel last + if image_data.ndim == 4: + image_data = torch.einsum('k h w c -> k c h w', image_data) + else: + image_data = torch.einsum('k t h w c -> k t c h w', image_data) + + for transform in self.transformations: + image_data = transform(image_data) + + action_data = ((action_data - self.norm_stats["action_min"]) / (self.norm_stats["action_max"] - self.norm_stats["action_min"])) * 2 - 1 + + qpos_data = (qpos_data - self.norm_stats["qpos_mean"]) / self.norm_stats["qpos_std"] + + sample = { + 'image': image_data, + 'state': qpos_data, + 'action': action_data, + 'is_pad': is_pad, + 'raw_lang': raw_lang, + 'reasoning': reasoning + } + assert raw_lang is not None, "" + if index == 0: + self.rank0_print(reasoning) + del image_data + del qpos_data + del action_data + del is_pad + del raw_lang + del reasoning + gc.collect() + torch.cuda.empty_cache() + + return self.llava_pythia_process.forward_process(sample, use_reasoning=self.data_args.use_reasoning) + # print(image_data.dtype, qpos_data.dtype, action_data.dtype, is_pad.dtype) + + +def get_norm_stats(dataset_path_list, action_dim, rank0_print=print): + all_qpos_data = [] + all_action_data = [] + all_episode_len = [] + + for dataset_path in dataset_path_list: + try: + with h5py.File(dataset_path, 'r') as root: + try: # only used for agelix and franka + qpos = root['/observations/qpos'][()] + action = root['/action'][()][:] + except: # for mobile aloha + if not root.get('/state/base_vel', None): + qpos = np.concatenate([ + root['/state/joint_position/left'][()][:-1], + root['/state/joint_position/right'][()][:-1], + # root['/state/base_vel'][()][:-1] + ], axis=1) + action = np.concatenate([ + root['/state/joint_position/left'][()][1:], + root['/state/joint_position/right'][()][1:], + # root['/action/base_vel'][()][:-1] + ], axis=1) + else: + qpos = np.concatenate([ + root['/state/joint_position/left'][()][:-1], + root['/state/joint_position/right'][()][:-1], + root['/state/base_vel'][()][:-1] + ], axis=1) + action = np.concatenate([ + root['/state/joint_position/left'][()][1:], + root['/state/joint_position/right'][()][1:], + root['/action/base_vel'][()][:-1] + ], axis=1) + qpos = qpos[:, :action_dim] + action = action[:, :action_dim] + except Exception as e: + rank0_print(f'Error loading {dataset_path} in get_norm_stats') + rank0_print(e) + quit() + all_qpos_data.append(torch.from_numpy(qpos)) + all_action_data.append(torch.from_numpy(action)) + all_episode_len.append(len(qpos)) + all_qpos_data = torch.cat(all_qpos_data, dim=0) + all_action_data = torch.cat(all_action_data, dim=0) + + # normalize action data + action_mean = all_action_data.mean(dim=[0]).float() + action_std = all_action_data.std(dim=[0]).float() + action_std = torch.clip(action_std, 1e-2, np.inf) # clipping + + # normalize qpos data + qpos_mean = all_qpos_data.mean(dim=[0]).float() + qpos_std = all_qpos_data.std(dim=[0]).float() + qpos_std = torch.clip(qpos_std, 1e-2, np.inf) # clipping + + action_min = all_action_data.min(dim=0).values.float() + action_max = all_action_data.max(dim=0).values.float() + + eps = 0.0001 + stats = {"action_mean": action_mean.numpy(), "action_std": action_std.numpy(), + "action_min": action_min.numpy() - eps,"action_max": action_max.numpy() + eps, + "qpos_mean": qpos_mean.numpy(), "qpos_std": qpos_std.numpy(), + "example_qpos": qpos} + + return stats, all_episode_len + +# calculating the norm stats corresponding to each kind of task (e.g. folding shirt, clean table....) +def get_norm_stats_by_tasks(dataset_path_list): + + data_tasks_dict = dict( + fold_shirt=[], + clean_table=[], + others=[], + ) + for dataset_path in dataset_path_list: + if 'fold' in dataset_path or 'shirt' in dataset_path: + key = 'fold_shirt' + elif 'clean_table' in dataset_path and 'pick' not in dataset_path: + key = 'clean_table' + else: + key = 'others' + data_tasks_dict[key].append(dataset_path) + + norm_stats_tasks = {k : None for k in data_tasks_dict.keys()} + + for k,v in data_tasks_dict.items(): + if len(v) > 0: + norm_stats_tasks[k], _ = get_norm_stats(v) + + return norm_stats_tasks + + +def find_all_hdf5(dataset_dir, skip_mirrored_data, rank0_print=print): + hdf5_files = [] + for root, dirs, files in os.walk(dataset_dir): + if 'pointcloud' in root: continue + for filename in fnmatch.filter(files, '*.hdf5'): + if 'features' in filename: continue + if skip_mirrored_data and 'mirror' in filename: + continue + hdf5_files.append(os.path.join(root, filename)) + if len(hdf5_files) == 0: + rank0_print(f"{RED} Found 0 hdf5 datasets found in {dataset_dir} {RESET}") + exit(0) + rank0_print(f'Found {len(hdf5_files)} hdf5 files') + return hdf5_files + +def BatchSampler(batch_size, episode_len_l, sample_weights): + sample_probs = np.array(sample_weights) / np.sum(sample_weights) if sample_weights is not None else None + sum_dataset_len_l = np.cumsum([0] + [np.sum(episode_len) for episode_len in episode_len_l]) + while True: + batch = [] + for _ in range(batch_size): + episode_idx = np.random.choice(len(episode_len_l), p=sample_probs) + step_idx = np.random.randint(sum_dataset_len_l[episode_idx], sum_dataset_len_l[episode_idx + 1]) + batch.append(step_idx) + yield batch + +def load_data(dataset_dir_l, name_filter, camera_names, batch_size_train, batch_size_val, chunk_size, config, action_dim, rank0_print=print, skip_mirrored_data=False, policy_class=None, stats_dir_l=None, sample_weights=None, train_ratio=0.99, return_dataset=False, llava_pythia_process=None): + if type(dataset_dir_l) == str: + dataset_dir_l = [dataset_dir_l] + dataset_path_list_list = [find_all_hdf5(dataset_dir, skip_mirrored_data, rank0_print=rank0_print) for dataset_dir in dataset_dir_l] + for d,dpl in zip(dataset_dir_l, dataset_path_list_list): + if len(dpl) == 0: + rank0_print("#2"*20) + rank0_print(d) + + num_episodes_0 = len(dataset_path_list_list[0]) + dataset_path_list = flatten_list(dataset_path_list_list) + dataset_path_list = [n for n in dataset_path_list if name_filter(n)] + num_episodes_l = [len(dataset_path_list) for dataset_path_list in dataset_path_list_list] + num_episodes_cumsum = np.cumsum(num_episodes_l) + + # obtain train test split on dataset_dir_l[0] + shuffled_episode_ids_0 = np.random.permutation(num_episodes_0) + train_episode_ids_0 = shuffled_episode_ids_0[:int(train_ratio * num_episodes_0)] + val_episode_ids_0 = shuffled_episode_ids_0[int(train_ratio * num_episodes_0):] + train_episode_ids_l = [train_episode_ids_0] + [np.arange(num_episodes) + num_episodes_cumsum[idx] for idx, num_episodes in enumerate(num_episodes_l[1:])] + val_episode_ids_l = [val_episode_ids_0] + #train_episode_ids_l = [] + #val_episode_ids_l = [] + #for idx, path_name in enumerate(dataset_path_list_list): + # num_episodes_i = len(dataset_path_list_list[idx]) + # shuffled_episode_ids_i = np.random.permutation(num_episodes_i) + # train_episode_ids_i = shuffled_episode_ids_i[:int(train_ratio * num_episodes_i)] + # val_episode_ids_i = shuffled_episode_ids_i[int(train_ratio * num_episodes_i):] + # train_episode_ids_l.append(train_episode_ids_i) + # val_episode_ids_l.append(val_episode_ids_i) + train_episode_ids = np.concatenate(train_episode_ids_l) + val_episode_ids = np.concatenate(val_episode_ids_l) + rank0_print(f'\n\nData from: {dataset_dir_l}\n- Train on {[len(x) for x in train_episode_ids_l]} episodes\n- Test on {[len(x) for x in val_episode_ids_l]} episodes\n\n') + + _, all_episode_len = get_norm_stats(dataset_path_list, action_dim) + rank0_print(f"{RED}All images: {sum(all_episode_len)}, Trajectories: {len(all_episode_len)} {RESET}") + train_episode_len_l = [[all_episode_len[i] for i in train_episode_ids] for train_episode_ids in train_episode_ids_l] + val_episode_len_l = [[all_episode_len[i] for i in val_episode_ids] for val_episode_ids in val_episode_ids_l] + + train_episode_len = flatten_list(train_episode_len_l) + val_episode_len = flatten_list(val_episode_len_l) + if stats_dir_l is None: + stats_dir_l = dataset_dir_l + elif type(stats_dir_l) == str: + stats_dir_l = [stats_dir_l] + + # calculate norm stats across all episodes + norm_stats, _ = get_norm_stats(flatten_list([find_all_hdf5(stats_dir, skip_mirrored_data, rank0_print=rank0_print) for stats_dir in stats_dir_l]), action_dim) + + # calculate norm stats corresponding to each kind of task + # norm_stats = get_norm_stats_by_tasks(flatten_list([find_all_hdf5(stats_dir, skip_mirrored_data, rank0_print=rank0_print) for stats_dir in stats_dir_l])) + rank0_print(f'Norm stats from: {[each.split("/")[-1] for each in stats_dir_l]}') + rank0_print(f'train_episode_len_l: {train_episode_len_l}') + + # print(f'train_episode_len: {train_episode_len}, val_episode_len: {val_episode_len}, train_episode_ids: {train_episode_ids}, val_episode_ids: {val_episode_ids}') + + robot = 'aloha' if config['action_head_args'].action_dim == 14 or ('aloha' in config['training_args'].output_dir) else 'franka' + # construct dataset and dataloader + train_dataset = EpisodicDataset(dataset_path_list, camera_names, norm_stats, train_episode_ids, train_episode_len, chunk_size, policy_class, robot=robot, llava_pythia_process=llava_pythia_process, data_args=config['data_args'], action_args=config['action_head_args']) + val_dataset = EpisodicDataset(dataset_path_list, camera_names, norm_stats, val_episode_ids, val_episode_len, chunk_size, policy_class, robot=robot, llava_pythia_process=llava_pythia_process, data_args=config['data_args'], action_args=config['action_head_args']) + + # print('EpisodicDataset .........') + # for i in range(100000): + # sample = train_dataset.__getitem__(i%1000) + # for k, v in sample.items(): + # if not isinstance(v, str): + # print(k) + # exit(0) + + sampler_params = { + 'train': {"batch_size": batch_size_train, 'episode_len_l': train_episode_len_l, 'sample_weights':sample_weights, 'episode_first': config['data_args'].episode_first}, + 'eval': {"batch_size": batch_size_val, 'episode_len_l': val_episode_len_l, 'sample_weights': None, 'episode_first': config['data_args'].episode_first} + } + + if return_dataset: + return train_dataset, val_dataset, norm_stats, sampler_params + + batch_sampler_train = BatchSampler(batch_size_train, train_episode_len_l, sample_weights) + batch_sampler_val = BatchSampler(batch_size_val, val_episode_len_l, None) + + train_num_workers = (8 if os.getlogin() == 'zfu' else 16) if train_dataset.augment_images else 2 + val_num_workers = 8 if train_dataset.augment_images else 2 + rank0_print(f'Augment images: {train_dataset.augment_images}, train_num_workers: {train_num_workers}, val_num_workers: {val_num_workers}') + train_dataloader = DataLoader(train_dataset, batch_sampler=batch_sampler_train, pin_memory=True, num_workers=train_num_workers, prefetch_factor=2) + val_dataloader = DataLoader(val_dataset, batch_sampler=batch_sampler_val, pin_memory=True, num_workers=val_num_workers, prefetch_factor=2) + + return train_dataloader, val_dataloader, norm_stats, train_dataset.is_sim + +def calibrate_linear_vel(base_action, c=None): + if c is None: + c = 0.0 # 0.19 + v = base_action[..., 0] + w = base_action[..., 1] + base_action = base_action.copy() + base_action[..., 0] = v - c * w + return base_action + +def smooth_base_action(base_action): + return np.stack([ + np.convolve(base_action[:, i], np.ones(5)/5, mode='same') for i in range(base_action.shape[1]) + ], axis=-1).astype(np.float32) + +def preprocess_base_action(base_action): + # base_action = calibrate_linear_vel(base_action) + base_action = smooth_base_action(base_action) + + return base_action + +def postprocess_base_action(base_action): + linear_vel, angular_vel = base_action + linear_vel *= 1.0 + angular_vel *= 1.0 + # angular_vel = 0 + # if np.abs(linear_vel) < 0.05: + # linear_vel = 0 + return np.array([linear_vel, angular_vel]) + +### env utils + +def sample_box_pose(): + x_range = [0.0, 0.2] + y_range = [0.4, 0.6] + z_range = [0.05, 0.05] + + ranges = np.vstack([x_range, y_range, z_range]) + cube_position = np.random.uniform(ranges[:, 0], ranges[:, 1]) + + cube_quat = np.array([1, 0, 0, 0]) + return np.concatenate([cube_position, cube_quat]) + +def sample_insertion_pose(): + # Peg + x_range = [0.1, 0.2] + y_range = [0.4, 0.6] + z_range = [0.05, 0.05] + + ranges = np.vstack([x_range, y_range, z_range]) + peg_position = np.random.uniform(ranges[:, 0], ranges[:, 1]) + + peg_quat = np.array([1, 0, 0, 0]) + peg_pose = np.concatenate([peg_position, peg_quat]) + + # Socket + x_range = [-0.2, -0.1] + y_range = [0.4, 0.6] + z_range = [0.05, 0.05] + + ranges = np.vstack([x_range, y_range, z_range]) + socket_position = np.random.uniform(ranges[:, 0], ranges[:, 1]) + + socket_quat = np.array([1, 0, 0, 0]) + socket_pose = np.concatenate([socket_position, socket_quat]) + + return peg_pose, socket_pose + +### helper functions + +def compute_dict_mean(epoch_dicts): + result = {k: None for k in epoch_dicts[0]} + num_items = len(epoch_dicts) + for k in result: + value_sum = 0 + for epoch_dict in epoch_dicts: + value_sum += epoch_dict[k] + result[k] = value_sum / num_items + return result + +def detach_dict(d): + new_d = dict() + for k, v in d.items(): + new_d[k] = v.detach() + return new_d + +def set_seed(seed): + torch.manual_seed(seed) + np.random.seed(seed) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/data_utils/__init__.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/data_utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/data_utils/check_data_integrity.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/data_utils/check_data_integrity.py new file mode 100644 index 00000000..10cbab05 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/data_utils/check_data_integrity.py @@ -0,0 +1,63 @@ +from dataset import find_all_hdf5, flatten_list +import os +path = "/media/rl/ADDS-4/" +import torch +import h5py +import numpy as np +from tqdm import tqdm +from PIL import Image +def get_norm_stats(dataset_path_list, rank0_print=print): + all_qpos_data = [] + all_action_data = [] + all_episode_len = [] + i = 0 + for dataset_path in tqdm(dataset_path_list): + try: + with h5py.File(dataset_path, 'r') as root: + qpos = root['/observations/qpos'][()] + qvel = root['/observations/qvel'][()] + if i % 5 == 0: + image = root['/observations/images']['cam_high'][(i*500+15) % 4000] + Image.fromarray(image).show() + + action = root['/action'][()] + except Exception as e: + rank0_print(f'Error loading {dataset_path} in get_norm_stats') + rank0_print(e) + all_qpos_data.append(torch.from_numpy(qpos)) + all_action_data.append(torch.from_numpy(action)) + all_episode_len.append(len(qpos)) + i += 1 + all_qpos_data = torch.cat(all_qpos_data, dim=0) + all_action_data = torch.cat(all_action_data, dim=0) + + # normalize action data + action_mean = all_action_data.mean(dim=[0]).float() + action_std = all_action_data.std(dim=[0]).float() + action_std = torch.clip(action_std, 1e-2, np.inf) # clipping + + # normalize qpos data + qpos_mean = all_qpos_data.mean(dim=[0]).float() + qpos_std = all_qpos_data.std(dim=[0]).float() + qpos_std = torch.clip(qpos_std, 1e-2, np.inf) # clipping + + action_min = all_action_data.min(dim=0).values.float() + action_max = all_action_data.max(dim=0).values.float() + + eps = 0.0001 + stats = {"action_mean": action_mean.numpy(), "action_std": action_std.numpy(), + "action_min": action_min.numpy() - eps,"action_max": action_max.numpy() + eps, + "qpos_mean": qpos_mean.numpy(), "qpos_std": qpos_std.numpy(), + "example_qpos": qpos} + + return stats, all_episode_len + + +################################################################################################################## +tasks = ["fold_two_shirts_wjj_03_21"] + +dataset_dir_l = [os.path.join(path, t) for t in tasks] +dataset_path_list_list = [find_all_hdf5(dataset_dir, skip_mirrored_data=True) for dataset_dir in dataset_dir_l] +dataset_path_list = flatten_list(dataset_path_list_list) + +print(get_norm_stats(dataset_path_list)) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/data_utils/data_collator.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/data_utils/data_collator.py new file mode 100644 index 00000000..dabfedd2 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/data_utils/data_collator.py @@ -0,0 +1,166 @@ +import copy +from dataclasses import dataclass, field, fields, asdict +import json +import logging +import pathlib +from typing import Dict, Optional, Sequence, List +import sys +import torch + +import transformers +import gc + +from PIL import Image +import numpy as np +import os +from qwen_vl_utils import process_vision_info +from qwen_vl_utils import fetch_image, fetch_video + +@dataclass +class DexVLADataCollatorForSupervisedDataset(object): + """Collate examples for supervised fine-tuning.""" + + multimodal_processor: transformers.AutoProcessor=None + computed_type: torch.dtype=None + tokenizer: transformers.AutoTokenizer=None + video: bool=False + + # @profile + def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]: + input_ids = [torch.flip(instance['input_ids'].squeeze(0), dims=[0]) for instance in instances] + attention_mask = [torch.flip(instance['attention_mask'].squeeze(0), dims=[0]) for instance in instances] + labels = [torch.flip(instance['labels'].squeeze(0), dims=[0]) for instance in instances] + raw_images = torch.stack([instances['raw_images'] for instances in instances]) + if self.video: + video_grid_thw = torch.stack([instances['video_grid_thw'] for instances in instances]) + pixel_values_videos = torch.stack([instances['pixel_values_videos'] for instances in instances]) + pixel_values = None + image_grid_thw=None + else: + image_grid_thw = torch.stack([instances['image_grid_thw'] for instances in instances]) + pixel_values = torch.stack([instances['pixel_values'] for instances in instances]) + pixel_values_videos = None + video_grid_thw = None + + labels = torch.nn.utils.rnn.pad_sequence(labels, + batch_first=True, + padding_value=-100) + labels = torch.flip(labels, dims=[1]) # left padding + input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, + batch_first=True, + padding_value=self.tokenizer.pad_token_id) + input_ids = torch.flip(input_ids, dims=[1]) + b = input_ids.shape[0] + if self.video: + video_grid_thw = video_grid_thw.reshape(b * video_grid_thw.shape[1], video_grid_thw.shape[2]) + pixel_values_videos = pixel_values_videos.reshape(b * pixel_values_videos.shape[1], pixel_values_videos.shape[2]) + + else: + image_grid_thw = image_grid_thw.reshape(b * image_grid_thw.shape[1], image_grid_thw.shape[2]) + pixel_values = pixel_values.reshape(b * pixel_values.shape[1], pixel_values.shape[2]) + + attention_mask = input_ids.ne(self.tokenizer.pad_token_id), + # attention_mask = torch.nn.utils.rnn.pad_sequence(labels, + # batch_first=True, + # padding_value=1) + + # max_length = max([each.shape[-1] for each in input_ids]) + # pad_id = self.tokenizer.pad_token_id + # for idx,_ in enumerate(input_ids): + # length = input_ids[idx].shape[-1] + # padd = torch.ones((1, max_length-length), dtype=torch.long, device=input_ids[idx].device) + # input_ids[idx] = torch.cat((padd*pad_id,input_ids[idx]), dim=-1) + # attention_mask[idx] = torch.cat((padd,attention_mask[idx]), dim=-1) + # labels[idx] = torch.cat((padd*-100,labels[idx]), dim=-1) + + if not isinstance(instances[0]['action'], torch.Tensor): + actions = torch.tensor(np.array([instance['action'] for instance in instances])) + states = torch.tensor(np.array([instance['state'] for instance in instances])) + else: + actions = torch.stack([instance['action'] for instance in instances]) + states = torch.stack([instance['state'] for instance in instances]) + + is_pad_all = torch.stack([instance['is_pad'] for instance in instances]) + + #print("#"*60) + #print(attention_mask.shape) + #exit(0) + batch = dict( + input_ids=input_ids, + # token_type_ids=model_inputs['token_type_ids'], + raw_images=raw_images, + attention_mask=attention_mask[0], + labels=labels, + image_grid_thw=image_grid_thw, + pixel_values_videos=pixel_values_videos, + actions=actions, + states=states, + video_grid_thw=video_grid_thw, + pixel_values=pixel_values, + is_pad=is_pad_all, + # attention_mask=input_ids.ne(temp_pad_token_id), + ) + del input_ids + del attention_mask + del labels + del pixel_values_videos + del pixel_values + del actions + del states + del video_grid_thw + del image_grid_thw + del is_pad_all + gc.collect() + torch.cuda.empty_cache() + return batch + + +@dataclass +class PaliGemmaVLADataCollatorForSupervisedDataset(object): + """Collate examples for supervised fine-tuning.""" + + multimodal_processor: transformers.AutoProcessor = None + computed_type: torch.dtype = None + + # @profile + def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]: + + prompt = "Task:" + raw_langs = [prompt + ins['raw_lang'] for ins in instances] + + images = torch.stack([ins['image'] for ins in instances]) + + answers = [ins['reasoning'] for ins in instances] + # answers = ["aaa" ,'bbb asdasda asda'] + model_inputs = self.multimodal_processor(text=raw_langs, suffix=answers, images=images, return_tensors="pt", padding="longest") + + pixel_values = copy.deepcopy(model_inputs['pixel_values']) + if not isinstance(instances[0]['action'], torch.Tensor): + actions = torch.tensor(np.array([instance['action'] for instance in instances])) + states = torch.tensor(np.array([instance['state'] for instance in instances])) + else: + actions = torch.stack([instance['action'] for instance in instances]) + states = torch.stack([instance['state'] for instance in instances]) + + is_pad_all = torch.stack([instance['is_pad'] for instance in instances]) + + batch = dict( + input_ids=model_inputs['input_ids'], + token_type_ids=model_inputs['token_type_ids'], + attention_mask=model_inputs['attention_mask'], + labels=model_inputs['labels'], + actions=actions, + states=states, + pixel_values=pixel_values, + is_pad=is_pad_all, + # attention_mask=input_ids.ne(temp_pad_token_id), + ) + + del model_inputs + del pixel_values + del actions + del states + del is_pad_all + gc.collect() + torch.cuda.empty_cache() + return batch diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/data_utils/dataset.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/data_utils/dataset.py new file mode 100644 index 00000000..61c80671 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/data_utils/dataset.py @@ -0,0 +1,509 @@ +import numpy as np +import torch +import os +import h5py +import pickle +import fnmatch +import cv2 +from time import time +from torch.utils.data import TensorDataset, DataLoader +import torchvision.transforms as transforms +from torchvision.transforms.functional import to_pil_image, to_tensor +import IPython +import copy +e = IPython.embed +from aloha_scripts.utils import * + +def flatten_list(l): + return [item for sublist in l for item in sublist] +import gc +class EpisodicDataset(torch.utils.data.Dataset): + def __init__(self, dataset_path_list, camera_names, norm_stats, episode_ids, episode_len, chunk_size, policy_class, robot=None, rank0_print=print, llava_pythia_process=None, data_args=None, action_args=None): + super(EpisodicDataset).__init__() + self.episode_ids = episode_ids + self.dataset_path_list = dataset_path_list + self.camera_names = camera_names + self.norm_stats = norm_stats + self.episode_len = episode_len + self.chunk_size = chunk_size + self.cumulative_len = np.cumsum(self.episode_len) + self.max_episode_len = max(episode_len) + self.policy_class = policy_class + self.llava_pythia_process = llava_pythia_process + self.data_args = data_args + self.action_args = action_args + self.robot = robot + self.rank0_print = rank0_print + + original_size = (480, 640) + new_size = eval(self.data_args.image_size_stable) # 320, 240 + new_size = (new_size[1], new_size[0]) + ratio = 0.95 + self.transformations = [ + # todo resize + transforms.Resize(size=original_size, antialias=True), + transforms.RandomCrop(size=[int(original_size[0] * ratio), int(original_size[1] * ratio)]), + transforms.Resize(original_size, antialias=True), + transforms.RandomRotation(degrees=[-5.0, 5.0], expand=False), + transforms.ColorJitter(brightness=0.3, contrast=0.4, saturation=0.5), # , hue=0.08) + transforms.Resize(size=new_size, antialias=True), + ] + + self.rank0_print(f"########################Current Image Size is [{self.data_args.image_size_stable}]###################################") + self.rank0_print(f"{RED}policy class: {self.policy_class}; augument: {True}{RESET}") + a=self.__getitem__(0) # initialize self.is_sim and self.transformations + if len(self.camera_names) > 2: + # self.rank0_print("%"*40) + self.rank0_print(f"The robot is {RED} {self.robot} {RESET} | The camera views: {RED} {self.camera_names} {RESET} | The history length: {RED} {self.data_args.history_images_length} {RESET}") + self.is_sim = False + + def __len__(self): + return sum(self.episode_len) + + def _locate_transition(self, index): + assert index < self.cumulative_len[-1] + episode_index = np.argmax(self.cumulative_len > index) # argmax returns first True index + start_ts = index - (self.cumulative_len[episode_index] - self.episode_len[episode_index]) + episode_id = self.episode_ids[episode_index] + return episode_id, start_ts + + def load_from_h5(self, dataset_path, start_ts): + + task_base_name = os.path.basename(dataset_path).replace('.hdf5', '') + task_dir_name = os.path.basename(os.path.dirname(dataset_path)) + + with h5py.File(dataset_path, 'r') as root: + + compressed = root.attrs.get('compress', False) + raw_lang = root['language_raw'][()].decode('utf-8') + reasonings = root['reasoning'][()] + reasoning = reasonings[start_ts].decode('utf-8') + # print(f"start_ts: {start_ts}") + # print(f"language_raw: {raw_lang}\n reasoning: {reasoning} \n") + + try: # only used for agelix and franka + qpos = root['/observations/qpos'][start_ts] + action = root['/action'][()][:, :] + except: # for mobile aloha + if not root.get('/state/base_vel', None): + qpos = np.concatenate([ + root['/state/joint_position/left'][()][:-1], + root['/state/joint_position/right'][()][:-1], + # root['/state/base_vel'][()][:-1] + ], + axis=1)[start_ts] + action = np.concatenate([ + root['/state/joint_position/left'][()][1:], + root['/state/joint_position/right'][()][1:], + # root['/action/base_vel'][()][:-1] + ], + axis=1) + else: + qpos = np.concatenate([ + root['/state/joint_position/left'][()][:-1], + root['/state/joint_position/right'][()][:-1], + root['/state/base_vel'][()][:-1]], + axis=1)[start_ts] + action = np.concatenate([ + root['/state/joint_position/left'][()][1:], + root['/state/joint_position/right'][()][1:], + root['/action/base_vel'][()][:-1]], + axis=1) + + # print(f'======debug qpos load h5: {qpos.shape}') + # print(f'======debug action load h5: {action.shape}') + qpos = qpos[:self.action_args.action_dim] + action = action[:, :self.action_args.action_dim] + # print(f'======debug qpos load h5 aft: {qpos.shape}') + # print(f'======debug action load h5 aft: {action.shape}') + original_action_shape = action.shape + episode_len = original_action_shape[0] + image_dict = dict() + for cam_name in self.camera_names: + image_dict[cam_name] = root[f'/observations/images/{cam_name}'][start_ts] + + if compressed: + for cam_name in image_dict.keys(): + decompressed_image = cv2.imdecode(image_dict[cam_name], 1) + image_dict[cam_name] = np.array(decompressed_image) + + # get all actions after and including start_ts + # action = action[start_ts:] # hack, to make timesteps more aligned + # action_len = episode_len - start_ts # hack, to make timesteps more aligned + action = action[max(0, start_ts - 1):] # hack, to make timesteps more aligned + action_len = episode_len - max(0, start_ts - 1) # hack, to make timesteps more aligned + return original_action_shape, action, action_len, image_dict, qpos, raw_lang, reasoning + def __getitem__(self, index): + episode_id, start_ts = self._locate_transition(index) + dataset_path = self.dataset_path_list[episode_id] + # print(dataset_path) + try: + original_action_shape, action, action_len, image_dict, qpos, raw_lang, reasoning = self.load_from_h5(dataset_path, start_ts) + except Exception as e: + print(f"Read {dataset_path} happens {YELLOW}{e}{RESET}") + try: + dataset_path = self.dataset_path_list[episode_id + 1] + except Exception as e: + dataset_path = self.dataset_path_list[episode_id - 1] + + original_action_shape, action, action_len, image_dict, qpos, raw_lang, reasoning = self.load_from_h5(dataset_path, start_ts) + + # self.is_sim = is_sim + padded_action = np.zeros((self.max_episode_len, original_action_shape[1]), dtype=np.float32) + + padded_action[:action_len] = action + is_pad = np.zeros(self.max_episode_len) + is_pad[action_len:] = 1 + + padded_action = padded_action[:self.chunk_size] + is_pad = is_pad[:self.chunk_size] + + # new axis for different cameras + all_cam_images = [] + for cam_name in self.camera_names: + all_cam_images.append(image_dict[cam_name]) + all_cam_images = np.stack(all_cam_images, axis=0) + + # construct observations + image_data = torch.from_numpy(all_cam_images) + qpos_data = torch.from_numpy(qpos).float() + action_data = torch.from_numpy(padded_action).float() + is_pad = torch.from_numpy(is_pad).bool() + + # if 'top' in self.camera_names or 'cam_high' in self.camera_names: # denote for data collect via bimanual UR5 + if self.robot == 'franka': + assert image_data.ndim==4, f"image_data's shape is {image_data.shape}, maybe the reason of adding historical images" + image_data = torch.stack([torch.from_numpy(cv2.cvtColor(img.numpy(), cv2.COLOR_BGR2RGB)) for img in image_data], dim=0) + + # channel last + if image_data.ndim == 4: + image_data = torch.einsum('k h w c -> k c h w', image_data) + else: + image_data = torch.einsum('k t h w c -> k t c h w', image_data) + + for transform in self.transformations: + image_data = transform(image_data) + + action_data = ((action_data - self.norm_stats["action_min"]) / (self.norm_stats["action_max"] - self.norm_stats["action_min"])) * 2 - 1 + + qpos_data = (qpos_data - self.norm_stats["qpos_mean"]) / self.norm_stats["qpos_std"] + + sample = { + 'image': image_data, + 'state': qpos_data, + 'action': action_data, + 'is_pad': is_pad, + 'raw_lang': raw_lang, + 'reasoning': reasoning + } + assert raw_lang is not None, "" + if index == 0: + self.rank0_print(reasoning) + del image_data + del qpos_data + del action_data + del is_pad + del raw_lang + del reasoning + gc.collect() + torch.cuda.empty_cache() + + return self.llava_pythia_process.forward_process(sample, use_reasoning=self.data_args.use_reasoning) + # print(image_data.dtype, qpos_data.dtype, action_data.dtype, is_pad.dtype) + + +def get_norm_stats(dataset_path_list, action_dim, rank0_print=print): + all_qpos_data = [] + all_action_data = [] + all_episode_len = [] + + for dataset_path in dataset_path_list: + try: + with h5py.File(dataset_path, 'r') as root: + try: # only used for agelix and franka + qpos = root['/observations/qpos'][()] + action = root['/action'][()][:] + except: # for mobile aloha + if not root.get('/state/base_vel', None): + qpos = np.concatenate([ + root['/state/joint_position/left'][()][:-1], + root['/state/joint_position/right'][()][:-1], + # root['/state/base_vel'][()][:-1] + ], axis=1) + action = np.concatenate([ + root['/state/joint_position/left'][()][1:], + root['/state/joint_position/right'][()][1:], + # root['/action/base_vel'][()][:-1] + ], axis=1) + else: + qpos = np.concatenate([ + root['/state/joint_position/left'][()][:-1], + root['/state/joint_position/right'][()][:-1], + root['/state/base_vel'][()][:-1] + ], axis=1) + action = np.concatenate([ + root['/state/joint_position/left'][()][1:], + root['/state/joint_position/right'][()][1:], + root['/action/base_vel'][()][:-1] + ], axis=1) + qpos = qpos[:, :action_dim] + action = action[:, :action_dim] + except Exception as e: + rank0_print(f'Error loading {dataset_path} in get_norm_stats') + rank0_print(e) + quit() + all_qpos_data.append(torch.from_numpy(qpos)) + all_action_data.append(torch.from_numpy(action)) + all_episode_len.append(len(qpos)) + all_qpos_data = torch.cat(all_qpos_data, dim=0) + all_action_data = torch.cat(all_action_data, dim=0) + + # normalize action data + action_mean = all_action_data.mean(dim=[0]).float() + action_std = all_action_data.std(dim=[0]).float() + action_std = torch.clip(action_std, 1e-2, np.inf) # clipping + + # normalize qpos data + qpos_mean = all_qpos_data.mean(dim=[0]).float() + qpos_std = all_qpos_data.std(dim=[0]).float() + qpos_std = torch.clip(qpos_std, 1e-2, np.inf) # clipping + + action_min = all_action_data.min(dim=0).values.float() + action_max = all_action_data.max(dim=0).values.float() + + eps = 0.0001 + stats = {"action_mean": action_mean.numpy(), "action_std": action_std.numpy(), + "action_min": action_min.numpy() - eps,"action_max": action_max.numpy() + eps, + "qpos_mean": qpos_mean.numpy(), "qpos_std": qpos_std.numpy(), + "example_qpos": qpos} + + return stats, all_episode_len + +# calculating the norm stats corresponding to each kind of task (e.g. folding shirt, clean table....) +def get_norm_stats_by_tasks(dataset_path_list): + + data_tasks_dict = dict( + fold_shirt=[], + clean_table=[], + others=[], + ) + for dataset_path in dataset_path_list: + if 'fold' in dataset_path or 'shirt' in dataset_path: + key = 'fold_shirt' + elif 'clean_table' in dataset_path and 'pick' not in dataset_path: + key = 'clean_table' + else: + key = 'others' + data_tasks_dict[key].append(dataset_path) + + norm_stats_tasks = {k : None for k in data_tasks_dict.keys()} + + for k,v in data_tasks_dict.items(): + if len(v) > 0: + norm_stats_tasks[k], _ = get_norm_stats(v) + + return norm_stats_tasks + + +def find_all_hdf5(dataset_dir, skip_mirrored_data, rank0_print=print): + hdf5_files = [] + for root, dirs, files in os.walk(dataset_dir): + if 'pointcloud' in root: continue + for filename in fnmatch.filter(files, '*.hdf5'): + if 'features' in filename: continue + if skip_mirrored_data and 'mirror' in filename: + continue + hdf5_files.append(os.path.join(root, filename)) + if len(hdf5_files) == 0: + rank0_print(f"{RED} Found 0 hdf5 datasets found in {dataset_dir} {RESET}") + exit(0) + rank0_print(f'Found {len(hdf5_files)} hdf5 files') + return hdf5_files + +def BatchSampler(batch_size, episode_len_l, sample_weights): + sample_probs = np.array(sample_weights) / np.sum(sample_weights) if sample_weights is not None else None + sum_dataset_len_l = np.cumsum([0] + [np.sum(episode_len) for episode_len in episode_len_l]) + while True: + batch = [] + for _ in range(batch_size): + episode_idx = np.random.choice(len(episode_len_l), p=sample_probs) + step_idx = np.random.randint(sum_dataset_len_l[episode_idx], sum_dataset_len_l[episode_idx + 1]) + batch.append(step_idx) + yield batch + +def load_data(dataset_dir_l, name_filter, camera_names, batch_size_train, batch_size_val, chunk_size, config, action_dim, rank0_print=print, skip_mirrored_data=False, policy_class=None, stats_dir_l=None, sample_weights=None, train_ratio=0.99, return_dataset=False, llava_pythia_process=None): + if type(dataset_dir_l) == str: + dataset_dir_l = [dataset_dir_l] + dataset_path_list_list = [find_all_hdf5(dataset_dir, skip_mirrored_data, rank0_print=rank0_print) for dataset_dir in dataset_dir_l] + for d,dpl in zip(dataset_dir_l, dataset_path_list_list): + if len(dpl) == 0: + rank0_print("#2"*20) + rank0_print(d) + + num_episodes_0 = len(dataset_path_list_list[0]) + dataset_path_list = flatten_list(dataset_path_list_list) + dataset_path_list = [n for n in dataset_path_list if name_filter(n)] + num_episodes_l = [len(dataset_path_list) for dataset_path_list in dataset_path_list_list] + num_episodes_cumsum = np.cumsum(num_episodes_l) + + # obtain train test split on dataset_dir_l[0] + shuffled_episode_ids_0 = np.random.permutation(num_episodes_0) + train_episode_ids_0 = shuffled_episode_ids_0[:int(train_ratio * num_episodes_0)] + val_episode_ids_0 = shuffled_episode_ids_0[int(train_ratio * num_episodes_0):] + train_episode_ids_l = [train_episode_ids_0] + [np.arange(num_episodes) + num_episodes_cumsum[idx] for idx, num_episodes in enumerate(num_episodes_l[1:])] + val_episode_ids_l = [val_episode_ids_0] + #train_episode_ids_l = [] + #val_episode_ids_l = [] + #for idx, path_name in enumerate(dataset_path_list_list): + # num_episodes_i = len(dataset_path_list_list[idx]) + # shuffled_episode_ids_i = np.random.permutation(num_episodes_i) + # train_episode_ids_i = shuffled_episode_ids_i[:int(train_ratio * num_episodes_i)] + # val_episode_ids_i = shuffled_episode_ids_i[int(train_ratio * num_episodes_i):] + # train_episode_ids_l.append(train_episode_ids_i) + # val_episode_ids_l.append(val_episode_ids_i) + train_episode_ids = np.concatenate(train_episode_ids_l) + val_episode_ids = np.concatenate(val_episode_ids_l) + rank0_print(f'\n\nData from: {dataset_dir_l}\n- Train on {[len(x) for x in train_episode_ids_l]} episodes\n- Test on {[len(x) for x in val_episode_ids_l]} episodes\n\n') + + _, all_episode_len = get_norm_stats(dataset_path_list, action_dim) + rank0_print(f"{RED}All images: {sum(all_episode_len)}, Trajectories: {len(all_episode_len)} {RESET}") + train_episode_len_l = [[all_episode_len[i] for i in train_episode_ids] for train_episode_ids in train_episode_ids_l] + val_episode_len_l = [[all_episode_len[i] for i in val_episode_ids] for val_episode_ids in val_episode_ids_l] + + train_episode_len = flatten_list(train_episode_len_l) + val_episode_len = flatten_list(val_episode_len_l) + if stats_dir_l is None: + stats_dir_l = dataset_dir_l + elif type(stats_dir_l) == str: + stats_dir_l = [stats_dir_l] + + # calculate norm stats across all episodes + norm_stats, _ = get_norm_stats(flatten_list([find_all_hdf5(stats_dir, skip_mirrored_data, rank0_print=rank0_print) for stats_dir in stats_dir_l]), action_dim) + + # calculate norm stats corresponding to each kind of task + # norm_stats = get_norm_stats_by_tasks(flatten_list([find_all_hdf5(stats_dir, skip_mirrored_data, rank0_print=rank0_print) for stats_dir in stats_dir_l])) + rank0_print(f'Norm stats from: {[each.split("/")[-1] for each in stats_dir_l]}') + rank0_print(f'train_episode_len_l: {train_episode_len_l}') + + # print(f'train_episode_len: {train_episode_len}, val_episode_len: {val_episode_len}, train_episode_ids: {train_episode_ids}, val_episode_ids: {val_episode_ids}') + + robot = 'aloha' if config['action_head_args'].action_dim == 14 or ('aloha' in config['training_args'].output_dir) else 'franka' + # construct dataset and dataloader + train_dataset = EpisodicDataset(dataset_path_list, camera_names, norm_stats, train_episode_ids, train_episode_len, chunk_size, policy_class, robot=robot, llava_pythia_process=llava_pythia_process, data_args=config['data_args'], action_args=config['action_head_args']) + val_dataset = EpisodicDataset(dataset_path_list, camera_names, norm_stats, val_episode_ids, val_episode_len, chunk_size, policy_class, robot=robot, llava_pythia_process=llava_pythia_process, data_args=config['data_args'], action_args=config['action_head_args']) + + # print('EpisodicDataset .........') + # for i in range(100000): + # sample = train_dataset.__getitem__(i%1000) + # for k, v in sample.items(): + # if not isinstance(v, str): + # print(k) + # exit(0) + + sampler_params = { + 'train': {"batch_size": batch_size_train, 'episode_len_l': train_episode_len_l, 'sample_weights':sample_weights, 'episode_first': config['data_args'].episode_first}, + 'eval': {"batch_size": batch_size_val, 'episode_len_l': val_episode_len_l, 'sample_weights': None, 'episode_first': config['data_args'].episode_first} + } + + if return_dataset: + return train_dataset, val_dataset, norm_stats, sampler_params + + batch_sampler_train = BatchSampler(batch_size_train, train_episode_len_l, sample_weights) + batch_sampler_val = BatchSampler(batch_size_val, val_episode_len_l, None) + + train_num_workers = (8 if os.getlogin() == 'zfu' else 16) if train_dataset.augment_images else 2 + val_num_workers = 8 if train_dataset.augment_images else 2 + rank0_print(f'Augment images: {train_dataset.augment_images}, train_num_workers: {train_num_workers}, val_num_workers: {val_num_workers}') + train_dataloader = DataLoader(train_dataset, batch_sampler=batch_sampler_train, pin_memory=True, num_workers=train_num_workers, prefetch_factor=2) + val_dataloader = DataLoader(val_dataset, batch_sampler=batch_sampler_val, pin_memory=True, num_workers=val_num_workers, prefetch_factor=2) + + return train_dataloader, val_dataloader, norm_stats, train_dataset.is_sim + +def calibrate_linear_vel(base_action, c=None): + if c is None: + c = 0.0 # 0.19 + v = base_action[..., 0] + w = base_action[..., 1] + base_action = base_action.copy() + base_action[..., 0] = v - c * w + return base_action + +def smooth_base_action(base_action): + return np.stack([ + np.convolve(base_action[:, i], np.ones(5)/5, mode='same') for i in range(base_action.shape[1]) + ], axis=-1).astype(np.float32) + +def preprocess_base_action(base_action): + # base_action = calibrate_linear_vel(base_action) + base_action = smooth_base_action(base_action) + + return base_action + +def postprocess_base_action(base_action): + linear_vel, angular_vel = base_action + linear_vel *= 1.0 + angular_vel *= 1.0 + # angular_vel = 0 + # if np.abs(linear_vel) < 0.05: + # linear_vel = 0 + return np.array([linear_vel, angular_vel]) + +### env utils + +def sample_box_pose(): + x_range = [0.0, 0.2] + y_range = [0.4, 0.6] + z_range = [0.05, 0.05] + + ranges = np.vstack([x_range, y_range, z_range]) + cube_position = np.random.uniform(ranges[:, 0], ranges[:, 1]) + + cube_quat = np.array([1, 0, 0, 0]) + return np.concatenate([cube_position, cube_quat]) + +def sample_insertion_pose(): + # Peg + x_range = [0.1, 0.2] + y_range = [0.4, 0.6] + z_range = [0.05, 0.05] + + ranges = np.vstack([x_range, y_range, z_range]) + peg_position = np.random.uniform(ranges[:, 0], ranges[:, 1]) + + peg_quat = np.array([1, 0, 0, 0]) + peg_pose = np.concatenate([peg_position, peg_quat]) + + # Socket + x_range = [-0.2, -0.1] + y_range = [0.4, 0.6] + z_range = [0.05, 0.05] + + ranges = np.vstack([x_range, y_range, z_range]) + socket_position = np.random.uniform(ranges[:, 0], ranges[:, 1]) + + socket_quat = np.array([1, 0, 0, 0]) + socket_pose = np.concatenate([socket_position, socket_quat]) + + return peg_pose, socket_pose + +### helper functions + +def compute_dict_mean(epoch_dicts): + result = {k: None for k in epoch_dicts[0]} + num_items = len(epoch_dicts) + for k in result: + value_sum = 0 + for epoch_dict in epoch_dicts: + value_sum += epoch_dict[k] + result[k] = value_sum / num_items + return result + +def detach_dict(d): + new_d = dict() + for k, v in d.items(): + new_d[k] = v.detach() + return new_d + +def set_seed(seed): + torch.manual_seed(seed) + np.random.seed(seed) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/data_utils/lerobot_dataset.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/data_utils/lerobot_dataset.py new file mode 100644 index 00000000..5e905f99 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/data_utils/lerobot_dataset.py @@ -0,0 +1,353 @@ + +import pickle +import fnmatch +import cv2 +cv2.setNumThreads(1) +from aloha_scripts.utils import * +import time +from torch.utils.data import TensorDataset, DataLoader +import torchvision.transforms as transforms +import os +import json +import numpy as np + +from aloha_scripts.lerobot_constants import TASK_CONFIGS + +from tqdm import tqdm +import torch + +from lerobot.common.datasets.lerobot_dataset import LeRobotDataset, LeRobotDatasetMetadata + +from typing import Protocol, SupportsIndex, TypeVar +T_co = TypeVar("T_co", covariant=True) +from tqdm import tqdm + + + + +class Dataset(Protocol[T_co]): + """Interface for a dataset with random access.""" + + def __getitem__(self, index: SupportsIndex) -> T_co: + raise NotImplementedError("Subclasses of Dataset should implement __getitem__.") + + def __len__(self) -> int: + raise NotImplementedError("Subclasses of Dataset should implement __len__.") + +class TransformedDataset(Dataset[T_co]): + def __init__(self, dataset: Dataset, norm_stats, camera_names,policy_class, robot=None, rank0_print=print, llava_pythia_process=None, data_args=None): + self._dataset = dataset + self.norm_stats = norm_stats + self.camera_names = camera_names + self.data_args = data_args + self.robot = robot + self.llava_pythia_process = llava_pythia_process + self.rank0_print = rank0_print + self.policy_class = policy_class + # augment images for training (default for dp and scaledp) + self.augment_images = True + + original_size = (480, 640) + new_size = eval(self.data_args.image_size_stable) # 320, 240 + new_size = (new_size[1], new_size[0]) + ratio = 0.95 + self.transformations = [ + # todo resize + # transforms.Resize(size=original_size, antialias=True), + transforms.RandomCrop(size=[int(original_size[0] * ratio), int(original_size[1] * ratio)]), + transforms.Resize(original_size, antialias=True), + transforms.RandomRotation(degrees=[-5.0, 5.0], expand=False), + transforms.ColorJitter(brightness=0.3, contrast=0.4, saturation=0.5), # , hue=0.08) + transforms.Resize(size=new_size, antialias=True), + ] + + if 'diffusion' in self.policy_class: + self.augment_images = True + else: + self.augment_images = False + + # self.rank0_print(f"########################Current Image Size is [{self.data_args.image_size_stable}]###################################") + # self.rank0_print(f"{RED}policy class: {self.policy_class}; augument: {self.augment_images}{RESET}") + # a=self.__getitem__(100) # initialize self.is_sim and self.transformations + # if len(self.camera_names) > 2: + # self.rank0_print("%"*40) + # self.rank0_print(f"The robot is {RED} {self.robot} {RESET} | The camera views: {RED} {self.camera_names} {RESET} | The history length: {RED} {self.data_args.history_images_length} {RESET}") + self.is_sim = False + + def __getitem__(self, index: SupportsIndex) -> T_co: + data = self._dataset[index] + + is_pad = data['action_is_pad'] + # sub_reason = data.meta. + + language_raw = self._dataset.meta.episodes[data['episode_index']]["language_dict"]['language_raw'] + if self.data_args.use_reasoning: + none_counter = 0 + for k in ['substep_reasonings', 'reason']: + vals = self._dataset.meta.episodes[data['episode_index']]["language_dict"][k] + if vals is not None: + if k == 'substep_reasonings': + sub_reasoning = vals[data['frame_index']] + else: + sub_reasoning = vals + # else: + # sub_reasoning = 'Next action:' + else: + none_counter += 1 + if none_counter == 2: + self.rank0_print(f"{RED} In {self._dataset.meta.repo_id}-{index}:{k} is None {RESET}") + + else: + sub_reasoning = 'Default outputs no reasoning' + + all_cam_images = [] + for cam_name in self.camera_names: + # Check if image is available + image = data[cam_name].numpy() + + # Transpose image to (height, width, channels) if needed + if image.shape[0] == 3: # If image is in (channels, height, width) + image = np.transpose(image, (1, 2, 0)) # Now it's (height, width, channels + + # image_dict[cam_name] = image # resize + + all_cam_images.append(image) + + all_cam_images = np.stack(all_cam_images, axis=0) + + # construct observations, and scale 0-1 to 0-255 + image_data = torch.from_numpy(all_cam_images) * 255 + image_data = image_data.to(dtype=torch.uint8) + # construct observations + qpos_data = data['observation.state'].float() + action_data = data['action'].float() + + # channel last + image_data = torch.einsum('k h w c -> k c h w', image_data) + + if self.augment_images: + for transform in self.transformations: + image_data = transform(image_data) + + norm_stats = self.norm_stats + if 'diffusion' in self.policy_class: + # normalize to [-1, 1] + action_data = ((action_data - norm_stats["action_min"]) / (norm_stats["action_max"] - norm_stats["action_min"])) * 2 - 1 + else: + # normalize to mean 0 std 1 + action_data = (action_data - norm_stats["action_mean"]) / norm_stats["action_std"] + + qpos_data = (qpos_data - norm_stats["qpos_mean"]) / norm_stats["qpos_std"] + + sample = { + 'image': image_data, + 'state': qpos_data, + 'action': action_data, + 'is_pad': is_pad, + 'raw_lang': language_raw, + 'reasoning': sub_reasoning + } + + return self.llava_pythia_process.forward_process(sample, use_reasoning=self.data_args.use_reasoning) + + def __len__(self) -> int: + return len(self._dataset) +def get_norm_stats(dataset_list): + """ + caculate all data action and qpos(robot state ) mean and std + """ + key_name_list=["observation.state","action"] + + all_qpos_data = [] + mean_list = [] + std_list = [] + length_list = [] + state_min_list = [] + state_max_list = [] + action_mean_list = [] + action_std_list = [] + action_max_list = [] + action_min_list = [] + + # Collect data from each dataset + for dataset in tqdm(dataset_list): + + mean_tensor = dataset.meta.stats["observation.state"]["mean"] + std_tensor = dataset.meta.stats["observation.state"]["std"] + state_max = dataset.meta.stats["observation.state"]["max"] + state_min = dataset.meta.stats["observation.state"]["min"] + + action_mean = dataset.meta.stats["action"]["mean"] + action_std = dataset.meta.stats["action"]["std"] + action_min = dataset.meta.stats["action"]["min"] + action_max = dataset.meta.stats["action"]["max"] + # Ensure the tensors are on CPU and convert to numpy arrays + mean_array = mean_tensor.cpu().numpy() if mean_tensor.is_cuda else mean_tensor.numpy() + std_array = std_tensor.cpu().numpy() if std_tensor.is_cuda else std_tensor.numpy() + state_max = state_max.cpu().numpy() if state_max.is_cuda else state_max.numpy() + state_min = state_min.cpu().numpy() if state_min.is_cuda else state_min.numpy() + + action_mean = action_mean.cpu().numpy() if action_mean.is_cuda else action_mean.numpy() + action_std = action_std.cpu().numpy() if action_std.is_cuda else action_std.numpy() + action_min = action_min.cpu().numpy() if action_min.is_cuda else action_min.numpy() + action_max = action_max.cpu().numpy() if action_max.is_cuda else action_max.numpy() + + # Append the arrays and the length of the dataset (number of samples) + mean_list.append(mean_array) + std_list.append(std_array) + state_max_list.append(state_max) + state_min_list.append(state_min) + action_mean_list.append(action_mean) + action_std_list.append(action_std) + action_max_list.append(action_max) + action_min_list.append(action_min) + + length_list.append(len(dataset)) # This is a single number, representing the number of samples + + # Convert lists to numpy arrays for easier manipulation + mean_array = np.array(mean_list) # Shape should be (num_datasets, 14) + std_array = np.array(std_list) # Shape should be (num_datasets, 14) + length_array = np.array(length_list) # Shape should be (num_datasets,) + + action_mean = np.array(action_mean_list) + action_std = np.array(action_std_list) + + state_max = np.max(state_max_list, axis=0) + state_min = np.min(state_min_list, axis=0) + action_max = np.max(action_max_list, axis=0) + action_min = np.min(action_min_list, axis=0) + + state_mean = np.sum(mean_array.T * length_array, axis=1) / np.sum(length_array) + + # To calculate the weighted variance (pooled variance): + + state_weighted_variance = np.sum(((length_array[:, None] - 1) * std_array ** 2 + (length_array[:, None] - 1) *mean_array**2),axis=0)/np.sum(length_array) - state_mean**2 + + # Calculate the overall standard deviation (square root of variance) + state_std = np.sqrt(state_weighted_variance) + + action_weighted_mean = np.sum(action_mean.T * length_array, axis=1) / np.sum(length_array) + action_weighted_variance = np.sum(((length_array[:, None] - 1) * action_std ** 2 + (length_array[:, None] - 1) *action_mean**2),axis=0)/np.sum(length_array) - action_weighted_mean**2 + action_weighted_std = np.sqrt(action_weighted_variance) + # Output the results + print(f"Overall Weighted Mean: {state_mean}") + print(f"Overall Weighted Std: {state_std}") + + eps = 0.0001 + stats = {"action_mean": action_weighted_mean, "action_std": action_weighted_std, + "action_min": action_min - eps, "action_max": action_max + eps, + "qpos_mean": state_mean, "qpos_std": state_std, + } + + + with open("stats.pkl", "wb") as f: + pickle.dump(stats, f) + all_episode_len = len(all_qpos_data) + return stats, all_episode_len + +def create_dataset(repo_id, chunk_size, home_lerobot=None, local_debug=False) -> Dataset: + with open(os.path.join(home_lerobot, repo_id, "meta", 'info.json'), 'r') as f: + data = json.load(f) + fps = data['fps'] + delta_timestamps = { + # "observation.state": [t / fps for t in range(args['chunk_size'])], + "action": [t / fps for t in range(chunk_size)], + } + + if local_debug: + print(f"{RED} Warning only using first two episodes {RESET}") + dataset = LeRobotDataset(repo_id, episodes=[0,1], delta_timestamps=delta_timestamps, local_files_only=True) + else: + dataset = LeRobotDataset(repo_id, delta_timestamps=delta_timestamps, local_files_only=True) + return dataset +def load_data(camera_names, chunk_size, config, rank0_print=print, policy_class=None, llava_pythia_process=None): + repo_id_list = TASK_CONFIGS[config['data_args'].task_name]['dataset_dir'] + dataset_list = [] + for repo_id in repo_id_list: + dataset = create_dataset(repo_id, chunk_size, home_lerobot=config['data_args'].home_lerobot, local_debug=config['training_args'].local_debug) + dataset_list.append(dataset) + norm_stats, all_episode_len = get_norm_stats(dataset_list) + train_dataset_list =[] + robot = 'aloha' if config['action_head_args'].action_dim == 14 or ('aloha' in config['training_args'].output_dir) else 'franka' + + rank0_print( + f"########################Current Image Size is [{config['data_args'].image_size_stable}]###################################") + rank0_print(f"{RED}policy class: {policy_class};{RESET}") + if len(camera_names) > 2: + # self.rank0_print("%"*40) + rank0_print( + f"The robot is {RED} {robot} {RESET} | The camera views: {RED} {camera_names} {RESET} | The history length: {RED} {config['data_args'].history_images_length} {RESET}") + + for dataset in dataset_list: + train_dataset_list.append(TransformedDataset( + dataset, norm_stats, camera_names, policy_class=policy_class, robot=robot, + rank0_print=rank0_print, llava_pythia_process=llava_pythia_process, data_args=config['data_args'])) + train_dataset = torch.utils.data.ConcatDataset(train_dataset_list) + # train_dataloder = DataLoader(train_dataset, batch_size=batch_size_train, shuffle=True, num_workers=8, pin_memory=True,prefetch_factor=2) + # val_dataloader = None + return train_dataset, None, norm_stats + +def get_norm_stats_by_tasks(dataset_path_list,args): + data_tasks_dict = dict( + fold_shirt=[], + clean_table=[], + others=[], + ) + for dataset_path in dataset_path_list: + if 'fold' in dataset_path or 'shirt' in dataset_path: + key = 'fold_shirt' + elif 'clean_table' in dataset_path and 'pick' not in dataset_path: + key = 'clean_table' + else: + key = 'others' + base_action = preprocess_base_action(base_action) + data_tasks_dict[key].append(dataset_path) + norm_stats_tasks = {k: None for k in data_tasks_dict.keys()} + for k, v in data_tasks_dict.items(): + if len(v) > 0: + norm_stats_tasks[k], _ = get_norm_stats(v) + return norm_stats_tasks + +def smooth_base_action(base_action): + return np.stack([ + np.convolve(base_action[:, i], np.ones(5) / 5, mode='same') for i in range(base_action.shape[1]) + ], axis=-1).astype(np.float32) + + +def preprocess_base_action(base_action): + # base_action = calibrate_linear_vel(base_action) + base_action = smooth_base_action(base_action) + + return base_action + + +def postprocess_base_action(base_action): + linear_vel, angular_vel = base_action + linear_vel *= 1.0 + angular_vel *= 1.0 + # angular_vel = 0 + # if np.abs(linear_vel) < 0.05: + # linear_vel = 0 + return np.array([linear_vel, angular_vel]) + +def compute_dict_mean(epoch_dicts): + result = {k: None for k in epoch_dicts[0]} + num_items = len(epoch_dicts) + for k in result: + value_sum = 0 + for epoch_dict in epoch_dicts: + value_sum += epoch_dict[k] + result[k] = value_sum / num_items + return result + + +def detach_dict(d): + new_d = dict() + for k, v in d.items(): + new_d[k] = v.detach() + return new_d + + +def set_seed(seed): + torch.manual_seed(seed) + np.random.seed(seed) \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/data_utils/truncate_data.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/data_utils/truncate_data.py new file mode 100644 index 00000000..69c8427b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/data_utils/truncate_data.py @@ -0,0 +1,158 @@ +""" +Example usage: +$ python3 script/compress_data.py --dataset_dir /scr/lucyshi/dataset/aloha_test +""" +import os +import h5py +import cv2 +import numpy as np +import argparse +from tqdm import tqdm + +# Constants +DT = 0.02 +JOINT_NAMES = ["waist", "shoulder", "elbow", "forearm_roll", "wrist_angle", "wrist_rotate"] +STATE_NAMES = JOINT_NAMES + ["gripper"] +TRUNCATE_LEN = 2250 + + +def compress_dataset(input_dataset_path, output_dataset_path): + # Check if output path exists + if os.path.exists(output_dataset_path): + print(f"The file {output_dataset_path} already exists. Exiting...") + return + + # Load the uncompressed dataset + with h5py.File(input_dataset_path, 'r') as infile: + # Create the compressed dataset + with h5py.File(output_dataset_path, 'w') as outfile: + + outfile.attrs['sim'] = infile.attrs['sim'] + outfile.attrs['compress'] = True + + # Copy non-image data directly + for key in infile.keys(): + if key != 'observations' and key != 'compress_len': + data = infile[key][:TRUNCATE_LEN] + out_data = outfile.create_dataset(key, (TRUNCATE_LEN, data.shape[1])) + out_data[:] = data + + data_compress_len = infile['compress_len'] + out_data_compress_len = outfile.create_dataset('compress_len', data_compress_len.shape) + out_data_compress_len[:] = data_compress_len + + # Create observation group in the output + obs_group = infile['observations'] + out_obs_group = outfile.create_group('observations') + for key in obs_group.keys(): + if key != 'images': + data = obs_group[key][:TRUNCATE_LEN] + out_data = out_obs_group.create_dataset(key, (TRUNCATE_LEN, data.shape[1])) + out_data[:] = data + + image_group = obs_group['images'] + out_image_group = out_obs_group.create_group('images') + + for cam_name in image_group.keys(): + data = image_group[cam_name][:TRUNCATE_LEN] + out_data = out_image_group.create_dataset(cam_name, (TRUNCATE_LEN, data.shape[1]), dtype='uint8') + out_data[:] = data + + + print(f"Truncated dataset saved to {output_dataset_path}") + + +def save_videos(video, dt, video_path=None): + if isinstance(video, list): + cam_names = list(video[0].keys()) + h, w, _ = video[0][cam_names[0]].shape + w = w * len(cam_names) + fps = int(1/dt) + out = cv2.VideoWriter(video_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h)) + # bitrate = 1000000 + # out.set(cv2.VIDEOWRITER_PROP_BITRATE, bitrate) + for ts, image_dict in enumerate(video): + images = [] + for cam_name in cam_names: + image = image_dict[cam_name] + image = image[:, :, [2, 1, 0]] # swap B and R channel + images.append(image) + images = np.concatenate(images, axis=1) + out.write(images) + out.release() + print(f'Saved video to: {video_path}') + elif isinstance(video, dict): + cam_names = list(video.keys()) + # Remove depth images + cam_names = [cam_name for cam_name in cam_names if '_depth' not in cam_name] + all_cam_videos = [] + for cam_name in cam_names: + all_cam_videos.append(video[cam_name]) + all_cam_videos = np.concatenate(all_cam_videos, axis=2) # width dimension + + n_frames, h, w, _ = all_cam_videos.shape + fps = int(1 / dt) + out = cv2.VideoWriter(video_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h)) + for t in range(n_frames): + image = all_cam_videos[t] + image = image[:, :, [2, 1, 0]] # swap B and R channel + out.write(image) + out.release() + print(f'Saved video to: {video_path}') + + +def load_and_save_first_episode_video(dataset_dir, video_path): + dataset_name = 'episode_0' + _, _, _, _, image_dict = load_hdf5(dataset_dir, dataset_name) + save_videos(image_dict, DT, video_path=video_path) + + +def load_hdf5(dataset_dir, dataset_name): + dataset_path = os.path.join(dataset_dir, dataset_name + '.hdf5') + if not os.path.isfile(dataset_path): + print(f'Dataset does not exist at \n{dataset_path}\n') + exit() + + with h5py.File(dataset_path, 'r') as root: + compressed = root.attrs.get('compress', False) + image_dict = dict() + for cam_name in root[f'/observations/images/'].keys(): + image_dict[cam_name] = root[f'/observations/images/{cam_name}'][()] + if compressed: + compress_len = root['/compress_len'][()] + + if compressed: + for cam_id, cam_name in enumerate(image_dict.keys()): + padded_compressed_image_list = image_dict[cam_name] + image_list = [] + for frame_id, padded_compressed_image in enumerate(padded_compressed_image_list): + image_len = int(compress_len[cam_id, frame_id]) + compressed_image = padded_compressed_image + image = cv2.imdecode(compressed_image, 1) + image_list.append(image) + image_dict[cam_name] = image_list + + return None, None, None, None, image_dict # Return only the image dict for this application + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Compress all HDF5 datasets in a directory.") + parser.add_argument('--dataset_dir', action='store', type=str, required=True, help='Directory containing the uncompressed datasets.') + + args = parser.parse_args() + + output_dataset_dir = args.dataset_dir + '_truncated' + os.makedirs(output_dataset_dir, exist_ok=True) + + # # Iterate over each file in the directory + # for filename in tqdm(os.listdir(args.dataset_dir), desc="Truncating data"): + # if filename.endswith('.hdf5'): + # input_path = os.path.join(args.dataset_dir, filename) + # output_path = os.path.join(output_dataset_dir, filename) + # compress_dataset(input_path, output_path) + # + # # After processing all datasets, load and save the video for the first episode + # print(f'Saving video for episode 0 in {output_dataset_dir}') + video_path = os.path.join(output_dataset_dir, 'episode_0_video.mp4') + load_and_save_first_episode_video(output_dataset_dir, video_path) + diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/deploy_policy.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/deploy_policy.py new file mode 100644 index 00000000..61ad4cd1 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/deploy_policy.py @@ -0,0 +1,265 @@ +import os.path + +from torchvision import transforms +from aloha_scripts.utils import * +import time +from data_utils.dataset import set_seed +from einops import rearrange +from transformers import AutoConfig, AutoProcessor, AutoTokenizer + +import sys +from policy.DexVLA.policy_heads.models.transformer_diffusion.modeling_dit_diffusion import * +from policy.DexVLA.policy_heads.models.transformer_diffusion.configuration_dit_diffusion import * +from policy.DexVLA.dex_vla.utils.image_processing_qwen2_vla import * +# from paligemma_vla.utils.processing_paligemma_vla import * +# from policy.DexVLA.dex_vla.utils.processing_qwen2_vla import * +from policy.DexVLA.evaluate.vla_policy import * +from policy.DexVLA.dex_vla.model_load_utils import load_model_for_eval +from PIL import Image +import pickle +import cv2 +import numpy as np + +def pre_process(robot_state_value, key, stats): + tmp = robot_state_value + tmp = (tmp - stats[key + '_mean']) / stats[key + '_std'] + return tmp + +def get_obs(deplot_env_obs, stats, camera_views=3): + ''' + 预处理输入到DexVLA的数据 + ''' + cur_top_rgb = deplot_env_obs['cam_high'] + cur_left_rgb = deplot_env_obs['cam_left'] + cur_right_rgb = deplot_env_obs['cam_right'] + # Change the data format donot Change the robot doudong action! + cur_top_rgb = cv2.cvtColor(cur_top_rgb, cv2.COLOR_BGRA2BGR)[:, :, ::-1] + cur_left_rgb = cv2.cvtColor(cur_left_rgb, cv2.COLOR_BGRA2BGR)[:, :, ::-1] + cur_right_rgb = cv2.cvtColor(cur_right_rgb, cv2.COLOR_BGRA2BGR)[:, :, ::-1] + + cur_joint_positions = deplot_env_obs['qpos'] + + cur_state_np = pre_process(cur_joint_positions, 'qpos', stats) + + cur_state = cur_state_np # deplot_env_obs['state'] + cur_state = np.expand_dims(cur_state, axis=0) + + # [2, 1, 128, 128, 3] + # [2, 480, 480, 3] + traj_rgb_np = np.array([cur_top_rgb, cur_left_rgb, cur_right_rgb]) + + traj_rgb_np = np.expand_dims(traj_rgb_np, axis=1) + traj_rgb_np = np.transpose(traj_rgb_np, (1, 0, 4, 2, 3)) + + # print("#" * 50) + # print(traj_rgb_np.shape) + + return cur_joint_positions, cur_state, traj_rgb_np + +class qwen2_vla_policy: + def __init__(self, policy_config, data_args=None): + super(qwen2_vla_policy).__init__() + self.load_policy(policy_config) + self.data_args = data_args + + def load_policy(self, policy_config): + self.policy_config = policy_config + model_base = policy_config["model_base"] if policy_config[ + 'enable_lora'] else None + model_path = policy_config["model_path"] + + self.tokenizer, self.policy, self.multimodal_processor, self.context_len = load_model_for_eval(model_path=model_path, + model_base=model_base, policy_config=policy_config) + self.tokenizer.add_special_tokens({'additional_special_tokens': ["[SOA]"]}) + + self.config = AutoConfig.from_pretrained('/'.join(model_path.split('/')[:-1]), trust_remote_code=True) + def datastruct_droid2qwen2vla(self, raw_lang): + messages = [ + { + "role": "user", + "content": [ + { + "type": "image", + "image": None, + }, + { + "type": "image", + "image": None, + }, + { + "type": "image", + "image": None, + }, + {"type": "text", "text": f""}, + ], + }, + # {"role": "assistant", "content": f''}, + ] + + messages[0]['content'][-1]['text'] = raw_lang + + return messages + def process_batch_to_qwen2_vla(self, curr_image, robo_state, raw_lang): + + if len(curr_image.shape) == 5: # 1,2,3,270,480 + curr_image = curr_image.squeeze(0) + + messages = self.datastruct_droid2qwen2vla(raw_lang) + image_data = torch.chunk(curr_image, curr_image.shape[0], dim=0) # top, left_wrist, right_wrist + image_list = [] + for i, each in enumerate(image_data): + ele = {} + each = Image.fromarray(each.cpu().squeeze(0).permute(1, 2, 0).numpy().astype(np.uint8)) + ele['image'] = each + ele['resized_height'] = 240 + ele['resized_width'] = 320 + + image_list.append(torch.from_numpy(np.array(each))) + # image_data = image_data / 255.0 + image_data = image_list + text = self.multimodal_processor.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True + ) + + video_inputs = None + model_inputs = self.multimodal_processor( + text=text, + images=image_data, + videos=video_inputs, + padding=True, + return_tensors="pt", + ) # After this code print the tensor image tensor dtype=torch.uint8)], 'videos': None} not recognized. + data_dict = dict(states=robo_state) + for k, v in model_inputs.items(): + data_dict[k] = v + return data_dict + + +class DexVLA: + def __init__(self, policy_config): + super(DexVLA).__init__() + self.policy_config = policy_config + self.policy = qwen2_vla_policy(policy_config) + print("Model Path: ", policy_config["model_path"]) + # self.image_list = [] + def get_action(self, obs): + # image_list = self.image_list + policy = self.policy + raw_lang = obs["raw_lang"] + policy_config = policy.policy_config + # eval 模式 + policy.policy.eval() + # 训练模型路径 + paths =policy_config["model_path"].split('/')[:-1] + # 存储统计信息。 + stats_path = policy_config["stats_path"] + with open(stats_path, 'rb') as f: + stats = pickle.load(f) + post_process = lambda a: ((a + 1) / 2) * (stats['action_max'] - stats['action_min']) + stats['action_min'] + ### Coding here + with torch.inference_mode(): + cur_state_np_raw, robot_state, traj_rgb_np = get_obs(obs, stats, camera_views=policy_config['camera_views']) + # 转换数据格式 + robot_state = torch.from_numpy(robot_state).float().cuda() + curr_image = torch.from_numpy(traj_rgb_np).float().cuda() + # print('rand crop resize is used!') + original_size = curr_image.shape[-2:] + ratio = 0.95 + curr_image = curr_image[..., + int(original_size[0] * (1 - ratio) / 2): int(original_size[0] * (1 + ratio) / 2), + int(original_size[1] * (1 - ratio) / 2): int(original_size[1] * (1 + ratio) / 2)] + curr_image = curr_image.squeeze(0) + resize_transform = transforms.Resize((240, 320), antialias=True) + curr_image = resize_transform(curr_image) + curr_image = curr_image.unsqueeze(0) + # image_list.append(curr_image) + batch = policy.process_batch_to_qwen2_vla(curr_image, robot_state, raw_lang) + all_actions, outputs = policy.policy.evaluate(**batch, is_eval=True, tokenizer=policy.tokenizer, + raw_images=curr_image) + # print(f"Output: {outputs}") + actions = all_actions.squeeze(0).cpu().to(dtype=torch.float32).numpy() + actions = post_process(actions) + # actions = actions[:25] + return actions + + +def compress_single_image(img): + """ + 将单张图片压缩为 JPEG 格式的二进制数据。 + + 参数: + img (numpy.ndarray): 输入的图像数组,形状为 (height, width, channels)。 + + 返回: + compressed_data (bytes): 压缩后的二进制数据。 + """ + # 使用 OpenCV 的 imencode 函数将图像编码为 JPEG 格式 + success, encoded_image = cv2.imencode('.jpg', img) + if not success: + raise ValueError("Failed to encode image") + + # 获取编码后的二进制数据 + compressed_data = encoded_image.tobytes() + + return compressed_data + +def encode_obs(observation): # Post-Process Observation + """ + Process input data for VLA model。 + """ + obs = observation + cam_high = obs["observation"]["head_camera"]["rgb"] + cam_left = obs["observation"]["left_camera"]["rgb"] + cam_right = obs["observation"]["right_camera"]["rgb"] + cam_high = compress_single_image(cam_high) + cam_left = compress_single_image(cam_left) + cam_right = compress_single_image(cam_right) + cam_high = cv2.imdecode(np.frombuffer(cam_high, np.uint8), cv2.IMREAD_COLOR) + cam_left = cv2.imdecode(np.frombuffer(cam_left, np.uint8), cv2.IMREAD_COLOR) + cam_right = cv2.imdecode(np.frombuffer(cam_right, np.uint8), cv2.IMREAD_COLOR) + qpos = (observation["joint_action"]["left_arm"] + [observation["joint_action"]["left_gripper"]] + + observation["joint_action"]["right_arm"] + [observation["joint_action"]["right_gripper"]]) + qpos = np.array(qpos) + return { + "cam_high": cam_high, + "cam_left": cam_left, + "cam_right": cam_right, + "qpos": qpos, + } + + +def get_model(usr_args): # from deploy_policy.yml and eval.sh (overrides) + policy_config = { + #### 1. Specify path to trained DexVLA(Required)############################# + "model_path": usr_args["model_path"], + "stats_path": usr_args["stats_path"], + "model_base": None, # only use for lora finetune + "enable_lora": False, # only use for lora finetune + "action_head": usr_args['action_head'], + "tinyvla": False, + "camera_views": 3, + "save_model": False, + } + model = DexVLA(policy_config) + return model # return your policy model + + +def eval(TASK_ENV, model, observation): + """ + All the function interfaces below are just examples + You can modify them according to your implementation + But we strongly recommend keeping the code logic unchanged + """ + obs = encode_obs(observation) # Post-Process Observation + instruction = TASK_ENV.get_instruction() + obs.update({"raw_lang": str(instruction)}) + actions = model.get_action(obs) # Get Action according to observation chunk + + for action in actions[:25]: # Execute each step of the action + # TASK_ENV.take_one_step_action(action) + TASK_ENV.take_action(action) + observation = TASK_ENV.get_obs() + + +def reset_model(model): # Clean the model cache at the beginning of every evaluation episode, such as the observation window + pass diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/deploy_policy.yml b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/deploy_policy.yml new file mode 100644 index 00000000..560ba9f0 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/deploy_policy.yml @@ -0,0 +1,13 @@ +# Basic experiment configuration (keep unchanged) +policy_name: DexVLA +task_name: task_name_you_test +task_config: null +ckpt_setting: null +seed: null +instruction_type: unseen + +# Specify path to trained DexVLA(Required) +model_path: "/Modify/policy/DexVLA/model/150000" +stats_path: "/Modify/robo2/policy/DexVLA/model/dataset_stats.pkl" +action_head: 'scale_dp_policy' # "dit_diffusion_policy" # or 'unet_diffusion_policy' +setting: NULL diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/__init__.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/__init__.py new file mode 100644 index 00000000..c9e87890 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/__init__.py @@ -0,0 +1,5 @@ +from .model_load_utils import * +from .train.dex_vla_trainer import * +from .models.modeling_dex_vla import * +from .models.configuration_dex_vla import * +from .utils.processing_qwen2_vla import * \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/external_vision_encoder/misc.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/external_vision_encoder/misc.py new file mode 100644 index 00000000..dc53c9d2 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/external_vision_encoder/misc.py @@ -0,0 +1,468 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Misc functions, including distributed helpers. + +Mostly copy-paste from torchvision references. +""" +import os +import subprocess +import time +from collections import defaultdict, deque +import datetime +import pickle +from packaging import version +from typing import Optional, List + +import torch +import torch.distributed as dist +from torch import Tensor + +# needed due to empty tensor bug in pytorch and torchvision 0.5 +import torchvision +if version.parse(torchvision.__version__) < version.parse('0.7'): + from torchvision.ops import _new_empty_tensor + from torchvision.ops.misc import _output_size + + +class SmoothedValue(object): + """Track a series of values and provide access to smoothed values over a + window or the global series average. + """ + + def __init__(self, window_size=20, fmt=None): + if fmt is None: + fmt = "{median:.4f} ({global_avg:.4f})" + self.deque = deque(maxlen=window_size) + self.total = 0.0 + self.count = 0 + self.fmt = fmt + + def update(self, value, n=1): + self.deque.append(value) + self.count += n + self.total += value * n + + def synchronize_between_processes(self): + """ + Warning: does not synchronize the deque! + """ + if not is_dist_avail_and_initialized(): + return + t = torch.tensor([self.count, self.total], dtype=torch.float64, device='cuda') + dist.barrier() + dist.all_reduce(t) + t = t.tolist() + self.count = int(t[0]) + self.total = t[1] + + @property + def median(self): + d = torch.tensor(list(self.deque)) + return d.median().item() + + @property + def avg(self): + d = torch.tensor(list(self.deque), dtype=torch.float32) + return d.mean().item() + + @property + def global_avg(self): + return self.total / self.count + + @property + def max(self): + return max(self.deque) + + @property + def value(self): + return self.deque[-1] + + def __str__(self): + return self.fmt.format( + median=self.median, + avg=self.avg, + global_avg=self.global_avg, + max=self.max, + value=self.value) + + +def all_gather(data): + """ + Run all_gather on arbitrary picklable data (not necessarily tensors) + Args: + data: any picklable object + Returns: + list[data]: list of data gathered from each rank + """ + world_size = get_world_size() + if world_size == 1: + return [data] + + # serialized to a Tensor + buffer = pickle.dumps(data) + storage = torch.ByteStorage.from_buffer(buffer) + tensor = torch.ByteTensor(storage).to("cuda") + + # obtain Tensor size of each rank + local_size = torch.tensor([tensor.numel()], device="cuda") + size_list = [torch.tensor([0], device="cuda") for _ in range(world_size)] + dist.all_gather(size_list, local_size) + size_list = [int(size.item()) for size in size_list] + max_size = max(size_list) + + # receiving Tensor from all ranks + # we pad the tensor because torch all_gather does not support + # gathering tensors of different shapes + tensor_list = [] + for _ in size_list: + tensor_list.append(torch.empty((max_size,), dtype=torch.uint8, device="cuda")) + if local_size != max_size: + padding = torch.empty(size=(max_size - local_size,), dtype=torch.uint8, device="cuda") + tensor = torch.cat((tensor, padding), dim=0) + dist.all_gather(tensor_list, tensor) + + data_list = [] + for size, tensor in zip(size_list, tensor_list): + buffer = tensor.cpu().numpy().tobytes()[:size] + data_list.append(pickle.loads(buffer)) + + return data_list + + +def reduce_dict(input_dict, average=True): + """ + Args: + input_dict (dict): all the values will be reduced + average (bool): whether to do average or sum + Reduce the values in the dictionary from all processes so that all processes + have the averaged results. Returns a dict with the same fields as + input_dict, after reduction. + """ + world_size = get_world_size() + if world_size < 2: + return input_dict + with torch.no_grad(): + names = [] + values = [] + # sort the keys so that they are consistent across processes + for k in sorted(input_dict.keys()): + names.append(k) + values.append(input_dict[k]) + values = torch.stack(values, dim=0) + dist.all_reduce(values) + if average: + values /= world_size + reduced_dict = {k: v for k, v in zip(names, values)} + return reduced_dict + + +class MetricLogger(object): + def __init__(self, delimiter="\t"): + self.meters = defaultdict(SmoothedValue) + self.delimiter = delimiter + + def update(self, **kwargs): + for k, v in kwargs.items(): + if isinstance(v, torch.Tensor): + v = v.item() + assert isinstance(v, (float, int)) + self.meters[k].update(v) + + def __getattr__(self, attr): + if attr in self.meters: + return self.meters[attr] + if attr in self.__dict__: + return self.__dict__[attr] + raise AttributeError("'{}' object has no attribute '{}'".format( + type(self).__name__, attr)) + + def __str__(self): + loss_str = [] + for name, meter in self.meters.items(): + loss_str.append( + "{}: {}".format(name, str(meter)) + ) + return self.delimiter.join(loss_str) + + def synchronize_between_processes(self): + for meter in self.meters.values(): + meter.synchronize_between_processes() + + def add_meter(self, name, meter): + self.meters[name] = meter + + def log_every(self, iterable, print_freq, header=None): + i = 0 + if not header: + header = '' + start_time = time.time() + end = time.time() + iter_time = SmoothedValue(fmt='{avg:.4f}') + data_time = SmoothedValue(fmt='{avg:.4f}') + space_fmt = ':' + str(len(str(len(iterable)))) + 'd' + if torch.cuda.is_available(): + log_msg = self.delimiter.join([ + header, + '[{0' + space_fmt + '}/{1}]', + 'eta: {eta}', + '{meters}', + 'time: {time}', + 'data: {data}', + 'max mem: {memory:.0f}' + ]) + else: + log_msg = self.delimiter.join([ + header, + '[{0' + space_fmt + '}/{1}]', + 'eta: {eta}', + '{meters}', + 'time: {time}', + 'data: {data}' + ]) + MB = 1024.0 * 1024.0 + for obj in iterable: + data_time.update(time.time() - end) + yield obj + iter_time.update(time.time() - end) + if i % print_freq == 0 or i == len(iterable) - 1: + eta_seconds = iter_time.global_avg * (len(iterable) - i) + eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) + if torch.cuda.is_available(): + print(log_msg.format( + i, len(iterable), eta=eta_string, + meters=str(self), + time=str(iter_time), data=str(data_time), + memory=torch.cuda.max_memory_allocated() / MB)) + else: + print(log_msg.format( + i, len(iterable), eta=eta_string, + meters=str(self), + time=str(iter_time), data=str(data_time))) + i += 1 + end = time.time() + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print('{} Total time: {} ({:.4f} s / it)'.format( + header, total_time_str, total_time / len(iterable))) + + +def get_sha(): + cwd = os.path.dirname(os.path.abspath(__file__)) + + def _run(command): + return subprocess.check_output(command, cwd=cwd).decode('ascii').strip() + sha = 'N/A' + diff = "clean" + branch = 'N/A' + try: + sha = _run(['git', 'rev-parse', 'HEAD']) + subprocess.check_output(['git', 'diff'], cwd=cwd) + diff = _run(['git', 'diff-index', 'HEAD']) + diff = "has uncommited changes" if diff else "clean" + branch = _run(['git', 'rev-parse', '--abbrev-ref', 'HEAD']) + except Exception: + pass + message = f"sha: {sha}, status: {diff}, branch: {branch}" + return message + + +def collate_fn(batch): + batch = list(zip(*batch)) + batch[0] = nested_tensor_from_tensor_list(batch[0]) + return tuple(batch) + + +def _max_by_axis(the_list): + # type: (List[List[int]]) -> List[int] + maxes = the_list[0] + for sublist in the_list[1:]: + for index, item in enumerate(sublist): + maxes[index] = max(maxes[index], item) + return maxes + + +class NestedTensor(object): + def __init__(self, tensors, mask: Optional[Tensor]): + self.tensors = tensors + self.mask = mask + + def to(self, device): + # type: (Device) -> NestedTensor # noqa + cast_tensor = self.tensors.to(device) + mask = self.mask + if mask is not None: + assert mask is not None + cast_mask = mask.to(device) + else: + cast_mask = None + return NestedTensor(cast_tensor, cast_mask) + + def decompose(self): + return self.tensors, self.mask + + def __repr__(self): + return str(self.tensors) + + +def nested_tensor_from_tensor_list(tensor_list: List[Tensor]): + # TODO make this more general + if tensor_list[0].ndim == 3: + if torchvision._is_tracing(): + # nested_tensor_from_tensor_list() does not export well to ONNX + # call _onnx_nested_tensor_from_tensor_list() instead + return _onnx_nested_tensor_from_tensor_list(tensor_list) + + # TODO make it support different-sized images + max_size = _max_by_axis([list(img.shape) for img in tensor_list]) + # min_size = tuple(min(s) for s in zip(*[img.shape for img in tensor_list])) + batch_shape = [len(tensor_list)] + max_size + b, c, h, w = batch_shape + dtype = tensor_list[0].dtype + device = tensor_list[0].device + tensor = torch.zeros(batch_shape, dtype=dtype, device=device) + mask = torch.ones((b, h, w), dtype=torch.bool, device=device) + for img, pad_img, m in zip(tensor_list, tensor, mask): + pad_img[: img.shape[0], : img.shape[1], : img.shape[2]].copy_(img) + m[: img.shape[1], :img.shape[2]] = False + else: + raise ValueError('not supported') + return NestedTensor(tensor, mask) + + +# _onnx_nested_tensor_from_tensor_list() is an implementation of +# nested_tensor_from_tensor_list() that is supported by ONNX tracing. +@torch.jit.unused +def _onnx_nested_tensor_from_tensor_list(tensor_list: List[Tensor]) -> NestedTensor: + max_size = [] + for i in range(tensor_list[0].dim()): + max_size_i = torch.max(torch.stack([img.shape[i] for img in tensor_list]).to(torch.float32)).to(torch.int64) + max_size.append(max_size_i) + max_size = tuple(max_size) + + # work around for + # pad_img[: img.shape[0], : img.shape[1], : img.shape[2]].copy_(img) + # m[: img.shape[1], :img.shape[2]] = False + # which is not yet supported in onnx + padded_imgs = [] + padded_masks = [] + for img in tensor_list: + padding = [(s1 - s2) for s1, s2 in zip(max_size, tuple(img.shape))] + padded_img = torch.nn.functional.pad(img, (0, padding[2], 0, padding[1], 0, padding[0])) + padded_imgs.append(padded_img) + + m = torch.zeros_like(img[0], dtype=torch.int, device=img.device) + padded_mask = torch.nn.functional.pad(m, (0, padding[2], 0, padding[1]), "constant", 1) + padded_masks.append(padded_mask.to(torch.bool)) + + tensor = torch.stack(padded_imgs) + mask = torch.stack(padded_masks) + + return NestedTensor(tensor, mask=mask) + + +def setup_for_distributed(is_master): + """ + This function disables printing when not in master process + """ + import builtins as __builtin__ + builtin_print = __builtin__.print + + def print(*args, **kwargs): + force = kwargs.pop('force', False) + if is_master or force: + builtin_print(*args, **kwargs) + + __builtin__.print = print + + +def is_dist_avail_and_initialized(): + if not dist.is_available(): + return False + if not dist.is_initialized(): + return False + return True + + +def get_world_size(): + if not is_dist_avail_and_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank(): + if not is_dist_avail_and_initialized(): + return 0 + return dist.get_rank() + + +def is_main_process(): + return get_rank() == 0 + + +def save_on_master(*args, **kwargs): + if is_main_process(): + torch.save(*args, **kwargs) + + +def init_distributed_mode(args): + if 'RANK' in os.environ and 'WORLD_SIZE' in os.environ: + args.rank = int(os.environ["RANK"]) + args.world_size = int(os.environ['WORLD_SIZE']) + args.gpu = int(os.environ['LOCAL_RANK']) + elif 'SLURM_PROCID' in os.environ: + args.rank = int(os.environ['SLURM_PROCID']) + args.gpu = args.rank % torch.cuda.device_count() + else: + print('Not using distributed mode') + args.distributed = False + return + + args.distributed = True + + torch.cuda.set_device(args.gpu) + args.dist_backend = 'nccl' + print('| distributed init (rank {}): {}'.format( + args.rank, args.dist_url), flush=True) + torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + torch.distributed.barrier() + setup_for_distributed(args.rank == 0) + + +@torch.no_grad() +def accuracy(output, target, topk=(1,)): + """Computes the precision@k for the specified values of k""" + if target.numel() == 0: + return [torch.zeros([], device=output.device)] + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +def interpolate(input, size=None, scale_factor=None, mode="nearest", align_corners=None): + # type: (Tensor, Optional[List[int]], Optional[float], str, Optional[bool]) -> Tensor + """ + Equivalent to nn.functional.interpolate, but with support for empty batch sizes. + This will eventually be supported natively by PyTorch, and this + class can go away. + """ + if version.parse(torchvision.__version__) < version.parse('0.7'): + if input.numel() > 0: + return torch.nn.functional.interpolate( + input, size, scale_factor, mode, align_corners + ) + + output_shape = _output_size(2, input, size, scale_factor) + output_shape = list(input.shape[:-2]) + list(output_shape) + return _new_empty_tensor(input, output_shape) + else: + return torchvision.ops.misc.interpolate(input, size, scale_factor, mode, align_corners) \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/external_vision_encoder/modules.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/external_vision_encoder/modules.py new file mode 100644 index 00000000..f340a74e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/external_vision_encoder/modules.py @@ -0,0 +1,207 @@ +import math +import abc +import numpy as np +import textwrap +from collections import OrderedDict + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torchvision import models as vision_models +from torchvision import transforms + + +class Module(torch.nn.Module): + """ + Base class for networks. The only difference from torch.nn.Module is that it + requires implementing @output_shape. + """ + @abc.abstractmethod + def output_shape(self, input_shape=None): + """ + Function to compute output shape from inputs to this module. + + Args: + input_shape (iterable of int): shape of input. Does not include batch dimension. + Some modules may not need this argument, if their output does not depend + on the size of the input, or if they assume fixed size input. + + Returns: + out_shape ([int]): list of integers corresponding to output shape + """ + raise NotImplementedError +""" +================================================ +Visual Backbone Networks +================================================ +""" +class ConvBase(Module): + """ + Base class for ConvNets. + """ + def __init__(self): + super(ConvBase, self).__init__() + + # dirty hack - re-implement to pass the buck onto subclasses from ABC parent + def output_shape(self, input_shape): + """ + Function to compute output shape from inputs to this module. + + Args: + input_shape (iterable of int): shape of input. Does not include batch dimension. + Some modules may not need this argument, if their output does not depend + on the size of the input, or if they assume fixed size input. + + Returns: + out_shape ([int]): list of integers corresponding to output shape + """ + raise NotImplementedError + + def forward(self, inputs): + x = self.nets(inputs) + if list(self.output_shape(list(inputs.shape)[1:])) != list(x.shape)[1:]: + raise ValueError('Size mismatch: expect size %s, but got size %s' % ( + str(self.output_shape(list(inputs.shape)[1:])), str(list(x.shape)[1:])) + ) + return x + +""" +================================================ +Pooling Networks +================================================ +""" +class SpatialSoftmax(ConvBase): + """ + Spatial Softmax Layer. + + Based on Deep Spatial Autoencoders for Visuomotor Learning by Finn et al. + https://rll.berkeley.edu/dsae/dsae.pdf + """ + def __init__( + self, + input_shape, + num_kp=32, + temperature=1., + learnable_temperature=False, + output_variance=False, + noise_std=0.0, + ): + """ + Args: + input_shape (list): shape of the input feature (C, H, W) + num_kp (int): number of keypoints (None for not using spatialsoftmax) + temperature (float): temperature term for the softmax. + learnable_temperature (bool): whether to learn the temperature + output_variance (bool): treat attention as a distribution, and compute second-order statistics to return + noise_std (float): add random spatial noise to the predicted keypoints + """ + super(SpatialSoftmax, self).__init__() + assert len(input_shape) == 3 + self._in_c, self._in_h, self._in_w = input_shape # (C, H, W) + + if num_kp is not None: + self.nets = torch.nn.Conv2d(self._in_c, num_kp, kernel_size=1) + self._num_kp = num_kp + else: + self.nets = None + self._num_kp = self._in_c + self.learnable_temperature = learnable_temperature + self.output_variance = output_variance + self.noise_std = noise_std + + if self.learnable_temperature: + # temperature will be learned + temperature = torch.nn.Parameter(torch.ones(1) * temperature, requires_grad=True) + self.register_parameter('temperature', temperature) + else: + # temperature held constant after initialization + temperature = torch.nn.Parameter(torch.ones(1) * temperature, requires_grad=False) + self.register_buffer('temperature', temperature) + + pos_x, pos_y = np.meshgrid( + np.linspace(-1., 1., self._in_w), + np.linspace(-1., 1., self._in_h) + ) + pos_x = torch.from_numpy(pos_x.reshape(1, self._in_h * self._in_w)).float() + pos_y = torch.from_numpy(pos_y.reshape(1, self._in_h * self._in_w)).float() + self.register_buffer('pos_x', pos_x) + self.register_buffer('pos_y', pos_y) + + self.kps = None + + def __repr__(self): + """Pretty print network.""" + header = format(str(self.__class__.__name__)) + return header + '(num_kp={}, temperature={}, noise={})'.format( + self._num_kp, self.temperature.item(), self.noise_std) + + def output_shape(self, input_shape): + """ + Function to compute output shape from inputs to this module. + + Args: + input_shape (iterable of int): shape of input. Does not include batch dimension. + Some modules may not need this argument, if their output does not depend + on the size of the input, or if they assume fixed size input. + + Returns: + out_shape ([int]): list of integers corresponding to output shape + """ + assert(len(input_shape) == 3) + assert(input_shape[0] == self._in_c) + return [self._num_kp, 2] + + def forward(self, feature): + """ + Forward pass through spatial softmax layer. For each keypoint, a 2D spatial + probability distribution is created using a softmax, where the support is the + pixel locations. This distribution is used to compute the expected value of + the pixel location, which becomes a keypoint of dimension 2. K such keypoints + are created. + + Returns: + out (torch.Tensor or tuple): mean keypoints of shape [B, K, 2], and possibly + keypoint variance of shape [B, K, 2, 2] corresponding to the covariance + under the 2D spatial softmax distribution + """ + + assert(feature.shape[1] == self._in_c) + assert(feature.shape[2] == self._in_h) + assert(feature.shape[3] == self._in_w) + if self.nets is not None: + feature = self.nets(feature) + + # [B, K, H, W] -> [B * K, H * W] where K is number of keypoints + feature = feature.reshape(-1, self._in_h * self._in_w) + # 2d softmax normalization + attention = F.softmax(feature / self.temperature, dim=-1) + # [1, H * W] x [B * K, H * W] -> [B * K, 1] for spatial coordinate mean in x and y dimensions + expected_x = torch.sum(self.pos_x * attention, dim=1, keepdim=True) + expected_y = torch.sum(self.pos_y * attention, dim=1, keepdim=True) + # stack to [B * K, 2] + expected_xy = torch.cat([expected_x, expected_y], 1) + # reshape to [B, K, 2] + feature_keypoints = expected_xy.view(-1, self._num_kp, 2) + + if self.training: + noise = torch.randn_like(feature_keypoints) * self.noise_std + feature_keypoints += noise + + if self.output_variance: + # treat attention as a distribution, and compute second-order statistics to return + expected_xx = torch.sum(self.pos_x * self.pos_x * attention, dim=1, keepdim=True) + expected_yy = torch.sum(self.pos_y * self.pos_y * attention, dim=1, keepdim=True) + expected_xy = torch.sum(self.pos_x * self.pos_y * attention, dim=1, keepdim=True) + var_x = expected_xx - expected_x * expected_x + var_y = expected_yy - expected_y * expected_y + var_xy = expected_xy - expected_x * expected_y + # stack to [B * K, 4] and then reshape to [B, K, 2, 2] where last 2 dims are covariance matrix + feature_covar = torch.cat([var_x, var_xy, var_xy, var_y], 1).reshape(-1, self._num_kp, 2, 2) + feature_keypoints = (feature_keypoints, feature_covar) + + if isinstance(feature_keypoints, tuple): + self.kps = (feature_keypoints[0].detach(), feature_keypoints[1].detach()) + else: + self.kps = feature_keypoints.detach() + return feature_keypoints + diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/external_vision_encoder/resnet_backbone.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/external_vision_encoder/resnet_backbone.py new file mode 100644 index 00000000..d58573ef --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/external_vision_encoder/resnet_backbone.py @@ -0,0 +1,79 @@ +from torch import nn +from torchvision.models._utils import IntermediateLayerGetter +from typing import Dict, List +import torchvision +import torch + +import torch.distributed as dist +def is_dist_avail_and_initialized(): + if not dist.is_available(): + return False + if not dist.is_initialized(): + return False + return True +def get_rank(): + if not is_dist_avail_and_initialized(): + return 0 + return dist.get_rank() +def is_main_process(): + return get_rank() == 0 +class FrozenBatchNorm2d(nn.Module): + # Implementation of FrozenBatchNorm2d, if not already provided + pass +class FrozenBatchNorm2d(nn.Module): + def __init__(self, n): + super(FrozenBatchNorm2d, self).__init__() + self.register_buffer('weight', torch.ones(n)) + self.register_buffer('bias', torch.zeros(n)) + self.register_buffer('running_mean', torch.zeros(n)) + self.register_buffer('running_var', torch.ones(n)) + + def forward(self, x): + if x.dim() != 4: + raise ValueError('expected 4D input (got {}D input)'.format(x.dim())) + scale = self.weight * self.running_var.rsqrt() + bias = self.bias - self.running_mean * scale + scale = scale.reshape(1, -1, 1, 1) + bias = bias.reshape(1, -1, 1, 1) + return x * scale + bias + +class BackboneBase(nn.Module): + + def __init__(self, backbone: nn.Module, train_backbone: bool, num_channels: int, return_interm_layers: bool): + super().__init__() + # for name, parameter in backbone.named_parameters(): # only train later layers # TODO do we want this? + # if not train_backbone or 'layer2' not in name and 'layer3' not in name and 'layer4' not in name: + # parameter.requires_grad_(False) + if return_interm_layers: + return_layers = {"layer1": "0", "layer2": "1", "layer3": "2", "layer4": "3"} + else: + return_layers = {'layer4': "0"} + self.body = IntermediateLayerGetter(backbone, return_layers=return_layers) + self.num_channels = num_channels + def forward(self, tensor): + xs = self.body(tensor) + # == key:0 + # resnet backbone size: torch.Size([16, 2048, 9, 15]) + # for k in xs.keys(): + # print(f'== key:{k}') + # print(f"resnet backbone size: {xs[k].size()}") + return xs['0'] +class Backbone(BackboneBase): + """ResNet backbone with frozen BatchNorm.""" + + def __init__(self, name: str, + train_backbone: bool, + return_interm_layers: bool, + dilation: bool): + backbone = getattr(torchvision.models, name)( + replace_stride_with_dilation=[False, False, dilation], + pretrained=False, + norm_layer=FrozenBatchNorm2d) # pretrained # TODO do we want frozen batch_norm?? + num_channels = 512 if name in ('resnet18', 'resnet34') else 2048 + super().__init__(backbone, train_backbone, num_channels, return_interm_layers) + +def build_backbone(args): + train_backbone = True + return_interm_layers = False #detr use False' + backbone = Backbone(args['backbone'], train_backbone, return_interm_layers, False) + return backbone \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/external_vision_encoder/resnet_film.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/external_vision_encoder/resnet_film.py new file mode 100644 index 00000000..dd87117c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/external_vision_encoder/resnet_film.py @@ -0,0 +1,463 @@ +from typing import Type, Any, Callable, Union, List, Mapping, Optional + +import copy +import torch +import torch.nn as nn +from torch import Tensor + + +def is_torch_version_lower_than_17(): + major_version = float(torch.__version__.split('.')[0]) + minor_version = float(torch.__version__.split('.')[1]) + return major_version == 1 and minor_version < 7 + + +if not is_torch_version_lower_than_17(): + # TODO: Make sure the torchvision version is similarly updated. + from torchvision.models import ResNet18_Weights, ResNet34_Weights, ResNet101_Weights, ResNet50_Weights + + +def conv3x3(in_planes: int, out_planes: int, stride: int = 1, groups: int = 1, dilation: int = 1) -> nn.Conv2d: + """3x3 convolution with padding""" + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=dilation, + groups=groups, + bias=False, + dilation=dilation, + ) + + +def conv1x1(in_planes: int, out_planes: int, stride: int = 1) -> nn.Conv2d: + """1x1 convolution""" + return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +class BasicBlock(nn.Module): + expansion: int = 1 + + def __init__( + self, + inplanes: int, + planes: int, + stride: int = 1, + downsample: Optional[nn.Module] = None, + groups: int = 1, + base_width: int = 64, + dilation: int = 1, + norm_layer: Optional[Callable[..., nn.Module]] = None, + ) -> None: + super().__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + if groups != 1 or base_width != 64: + raise ValueError("BasicBlock only supports groups=1 and base_width=64") + if dilation > 1: + raise NotImplementedError("Dilation > 1 not supported in BasicBlock") + # Both self.conv1 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = norm_layer(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = norm_layer(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x: Tensor, film_features: Optional[Tensor] = None) -> Tensor: + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + # Apply FiLM here + if film_features is not None: + # gamma, beta will be (B, 1, 1, planes) + gamma, beta = torch.split(film_features, 1, dim=1) + gamma = gamma.squeeze().view(x.size(0), -1, 1, 1) + beta = beta.squeeze().view(x.size(0), -1, 1, 1) + out = (1 + gamma) * out + beta + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2) + # while original implementation places the stride at the first 1x1 convolution(self.conv1) + # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385. + # This variant is also known as ResNet V1.5 and improves accuracy according to + # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch. + + expansion: int = 4 + + def __init__( + self, + inplanes: int, + planes: int, + stride: int = 1, + downsample: Optional[nn.Module] = None, + groups: int = 1, + base_width: int = 64, + dilation: int = 1, + norm_layer: Optional[Callable[..., nn.Module]] = None, ) -> None: + super().__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + width = int(planes * (base_width / 64.0)) * groups + # Both self.conv2 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv1x1(inplanes, width) + self.bn1 = norm_layer(width) + self.conv2 = conv3x3(width, width, stride, groups, dilation) + self.bn2 = norm_layer(width) + self.conv3 = conv1x1(width, planes * self.expansion) + self.bn3 = norm_layer(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x: Tensor) -> Tensor: + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + + out = self.relu(out) + + return out + + +class ResNetWithExtraModules(nn.Module): + """Update standard ResNet image classification models with FiLM.""" + + def __init__( + self, + block: Type[Union[BasicBlock, Bottleneck]], + layers: List[int], + num_classes: int = 1000, + zero_init_residual: bool = False, + groups: int = 1, + width_per_group: int = 64, + replace_stride_with_dilation: Optional[List[bool]] = None, + norm_layer: Optional[Callable[..., nn.Module]] = None, + film_config: Optional[Mapping[str, Any]] = None, ) -> None: + super().__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + self._norm_layer = norm_layer + + # Save how many blocks in each layer + self.layers = layers + + # FiLM only implemented for BasicBlock for now + self.use_film = film_config is not None and film_config['use'] + if self.use_film: + self.film_config = film_config + self.film_planes = film_config['film_planes'] + self.expansion = block.expansion + + self.inplanes = 64 + self.dilation = 1 + if replace_stride_with_dilation is None: + # each element in the tuple indicates if we should replace + # the 2x2 stride with a dilated convolution instead + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError( + "replace_stride_with_dilation should be None " + f"or a 3-element tuple, got {replace_stride_with_dilation}" + ) + + in_channels_conv1 = 4 if ( + film_config is not None and + film_config.get('append_object_mask', None) is not None) else 3 + + self.groups = groups + self.base_width = width_per_group + self.conv1 = nn.Conv2d(in_channels_conv1, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False) + self.bn1 = norm_layer(self.inplanes) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, 256, layers[0]) + self.layer2 = self._make_layer(block, 512, layers[1], stride=2, dilate=replace_stride_with_dilation[0]) + self.layer3 = self._make_layer(block, 1024, layers[2], stride=2, dilate=replace_stride_with_dilation[1]) + self.layer4 = self._make_layer(block, 2048, layers[3], stride=2, dilate=replace_stride_with_dilation[2]) + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.fc = nn.Linear(512 * block.expansion, num_classes) + + for m_name, m in self.named_modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu") + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck) and m.bn3.weight is not None: + nn.init.constant_(m.bn3.weight, 0) # type: ignore[arg-type] + elif isinstance(m, BasicBlock) and m.bn2.weight is not None: + nn.init.constant_(m.bn2.weight, 0) # type: ignore[arg-type] + + def _make_layer( + self, + block: Type[Union[BasicBlock, Bottleneck]], + planes: int, + blocks: int, + stride: int = 1, + dilate: bool = False, ) -> nn.Sequential: + norm_layer = self._norm_layer + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + norm_layer(planes * block.expansion), + ) + + layers = [ + block(self.inplanes, planes, stride, downsample, self.groups, self.base_width, previous_dilation, + norm_layer, ) + ] + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append( + block( + self.inplanes, + planes, + groups=self.groups, + base_width=self.base_width, + dilation=self.dilation, + norm_layer=norm_layer, + ) + ) + + if self.use_film: + return nn.ModuleList(layers) + else: + return nn.Sequential(*layers) + + def _forward_impl_film(self, x: Tensor, film_features: List[Optional[Tensor]], flatten: bool = True): + assert self.use_film and film_features is not None + + def _extract_film_features_for_layer(film_feat: Optional[Tensor], layer_idx: int): + if film_features[layer_idx] is None: + return [None] * self.layers[layer_idx] + + num_planes = self.film_planes[layer_idx] + num_blocks = self.layers[layer_idx] + film_feat = film_feat.view(-1, 2, num_blocks, num_planes) + film_feat_per_block = torch.split(film_feat, 1, dim=2) + return film_feat_per_block + + for layer_idx, layer in enumerate([self.layer1, self.layer2, self.layer3, self.layer4]): + film_feat_per_block = _extract_film_features_for_layer( + film_features[layer_idx], layer_idx) + for block_idx, block in enumerate(layer): + if film_feat_per_block[block_idx] is not None: + assert x.shape[0] == film_feat_per_block[block_idx].shape[0], ('FiLM batch size does not match') + x = block(x, film_features=film_feat_per_block[block_idx]) + + x = self.avgpool(x) + if flatten: + x = torch.flatten(x, 1) + x = self.fc(x) + return x + + def _forward_impl(self, + x: Tensor, + film_features: List[Optional[Tensor]], + flatten: bool = True) -> Tensor: + # See note [TorchScript super()] + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + if self.use_film: + return self._forward_impl_film(x, film_features, flatten=flatten) + else: + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.avgpool(x) + if flatten: + x = torch.flatten(x, 1) + x = self.fc(x) + + return x + + def forward(self, + x: Tensor, + film_features: List[Optional[Tensor]], **kwargs) -> Tensor: + return self._forward_impl(x, film_features, **kwargs) + + +def _resnet( + block: Type[Union[BasicBlock, Bottleneck]], + layers: List[int], + weights, + progress: bool, + **kwargs: Any, +) -> ResNetWithExtraModules: + model_kwargs = copy.deepcopy(kwargs) + if 'pretrained' in model_kwargs: + del model_kwargs['pretrained'] + if 'arch' in model_kwargs: + del model_kwargs['arch'] + model = ResNetWithExtraModules(block, layers, **model_kwargs) + + if weights is not None: + model.load_state_dict(weights.get_state_dict(progress=progress)) + elif kwargs.get('pretrained', False) and kwargs.get('arch') is not None: + if float(torch.__version__.split('.')[1]) < 7: + # Copied from https://pytorch.org/vision/0.11/_modules/torchvision/models/resnet.html#resnet18 + model_urls = { + 'resnet18': 'https://download.pytorch.org/models/resnet18-f37072fd.pth', + 'resnet34': 'https://download.pytorch.org/models/resnet34-b627a593.pth', + 'resnet50': 'https://download.pytorch.org/models/resnet50-0676ba61.pth', + 'resnet101': 'https://download.pytorch.org/models/resnet101-63fe2227.pth', + 'resnet152': 'https://download.pytorch.org/models/resnet152-394f9c45.pth', + 'resnext50_32x4d': 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth', + 'resnext101_32x8d': 'https://download.pytorch.org/models/resnext101_32x8d-8ba56ff5.pth', + 'wide_resnet50_2': 'https://download.pytorch.org/models/wide_resnet50_2-95faca4d.pth', + 'wide_resnet101_2': 'https://download.pytorch.org/models/wide_resnet101_2-32ee1156.pth', + } + + # state_dict = load_state_dict_from_url(model_urls[arch], + # progress=progress) + state_dict = torch.hub.load_state_dict_from_url(model_urls[kwargs.get('arch')], + progress=progress) + model.load_state_dict(state_dict) + + return model + + +def resnet18(*, weights=None, progress: bool = True, **kwargs: Any) -> ResNetWithExtraModules: + """ResNet-18 from `Deep Residual Learning for Image Recognition `__. + + Args: + weights (:class:`~torchvision.models.ResNet18_Weights`, optional): The + pretrained weights to use. See + :class:`~torchvision.models.ResNet18_Weights` below for + more details, and possible values. By default, no pre-trained + weights are used. + progress (bool, optional): If True, displays a progress bar of the + download to stderr. Default is True. + **kwargs: parameters passed to the ``torchvision.models.resnet.ResNet`` + base class. Please refer to the `source code + `_ + for more details about this class. + + .. autoclass:: torchvision.models.ResNet18_Weights + :members: + """ + if is_torch_version_lower_than_17(): + kwargs["arch"] = "resnet18" + weights = None + else: + weights = ResNet18_Weights.verify(weights) + + return _resnet(BasicBlock, [2, 2, 2, 2], weights, progress, **kwargs) + + +def resnet34(*, weights=None, progress: bool = True, **kwargs: Any) -> ResNetWithExtraModules: + """ResNet-34 from `Deep Residual Learning for Image Recognition `__. + + Args: + weights (:class:`~torchvision.models.ResNet34_Weights`, optional): The + pretrained weights to use. See + :class:`~torchvision.models.ResNet34_Weights` below for + more details, and possible values. By default, no pre-trained + weights are used. + progress (bool, optional): If True, displays a progress bar of the + download to stderr. Default is True. + **kwargs: parameters passed to the ``torchvision.models.resnet.ResNet`` + base class. Please refer to the `source code + `_ + for more details about this class. + + .. autoclass:: torchvision.models.ResNet34_Weights + :members: + """ + if is_torch_version_lower_than_17(): + kwargs["arch"] = "resnet34" + weights = None + else: + weights = ResNet34_Weights.verify(weights) + + return _resnet(BasicBlock, [3, 4, 6, 3], weights, progress, **kwargs) + + +def resnet50(*, weights=None, progress: bool = True, **kwargs: Any) -> ResNetWithExtraModules: + """Res 50 from `Deep Residual Learning for Image Recognition `__.""" + if is_torch_version_lower_than_17(): + kwargs["arch"] = "resnet50" + weights = None + else: + weights = ResNet50_Weights.verify(weights) + return _resnet(BasicBlock, [3, 4, 6, 3], weights, progress, **kwargs) + + +def resnet101(*, weights=None, progress: bool = True, **kwargs: Any) -> ResNetWithExtraModules: + """ResNet-101 from `Deep Residual Learning for Image Recognition `__. + + .. note:: + The bottleneck of TorchVision places the stride for downsampling to the second 3x3 + convolution while the original paper places it to the first 1x1 convolution. + This variant improves the accuracy and is known as `ResNet V1.5 + `_. + + Args: + weights (:class:`~torchvision.models.ResNet101_Weights`, optional): The + pretrained weights to use. See + :class:`~torchvision.models.ResNet101_Weights` below for + more details, and possible values. By default, no pre-trained + weights are used. + progress (bool, optional): If True, displays a progress bar of the + download to stderr. Default is True. + **kwargs: parameters passed to the ``torchvision.models.resnet.ResNet`` + base class. Please refer to the `source code + `_ + for more details about this class. + + .. autoclass:: torchvision.models.ResNet101_Weights + :members: + """ + if is_torch_version_lower_than_17(): + kwargs["arch"] = "resnet101" + weights = None + else: + weights = ResNet101_Weights.verify(weights) + return _resnet(Bottleneck, [3, 4, 23, 3], weights, progress, **kwargs) \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/external_vision_encoder/resnet_vision_encoder.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/external_vision_encoder/resnet_vision_encoder.py new file mode 100644 index 00000000..1e1b7128 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/external_vision_encoder/resnet_vision_encoder.py @@ -0,0 +1,73 @@ +import torch.nn as nn +from .resnet_backbone import build_backbone +from .modules import SpatialSoftmax +import numpy as np +import torch + +class ResNetEncoder(nn.Module): + def __init__(self, len_cameras=3, use_film=False): + super().__init__() + backbones = [] + pools = [] + linears = [] + img_fea_dim = stsm_num_kp = 512 + self.len_cameras = len_cameras + self.use_film = use_film + self.backbone_name = 'resnet50' + for _ in range(len_cameras): + backbone = build_backbone({"backbone": "resnet50"}) + backbones.append(backbone) + + input_shape = [2048, 8, 10] + + pools.append( + nn.Sequential( + SpatialSoftmax(**{'input_shape': input_shape, 'num_kp': stsm_num_kp, 'temperature': 1.0, + 'learnable_temperature': False, 'noise_std': 0.0}), + nn.Flatten(start_dim=1, end_dim=-1) + ) + ) + linears.append( + nn.Sequential( + nn.Linear(int(np.prod([stsm_num_kp, 2])), stsm_num_kp), + nn.ReLU(), + nn.Linear(stsm_num_kp, img_fea_dim) + ) + ) + + self.backbones = nn.ModuleList(backbones) + self.pools = nn.ModuleList(pools) + self.linears = nn.ModuleList(linears) + self.projection = nn.Sequential( + nn.Linear(len_cameras * 512, 768), + nn.ReLU(), + nn.Linear(768, 768), + ) + + def forward(self, images, lang_embed=None): + all_cam_features = [] + images = (images / 255.0).to(torch.bfloat16) + for cam_id in range(self.len_cameras): + if self.use_film and lang_embed is not None: + cur_img = images[:, cam_id] + + # if self.color_randomizer is not None: + # cur_img = self.color_randomizer._forward_in(cur_img) + + + features = self.backbones[cam_id](cur_img, lang_embed) + + else: + cur_img = images[:, cam_id] + # if self.color_randomizer is not None: + # cur_img = self.color_randomizer._forward_in(cur_img) + features = self.backbones[cam_id](cur_img) + + pool_features = self.pools[cam_id]( + features).to(torch.bfloat16) + out_features = self.linears[cam_id](pool_features) + + all_cam_features.append(out_features) + obs_cond = torch.cat(all_cam_features, dim=1) + obs_cond = self.projection(obs_cond) + return obs_cond diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/model_load_utils.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/model_load_utils.py new file mode 100644 index 00000000..4d0d94ce --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/model_load_utils.py @@ -0,0 +1,633 @@ +import torch + + +import transformers +import copy +from dataclasses import dataclass, field, fields, asdict +import json +import logging +import pathlib +from typing import Dict, Optional, Sequence, List +from transformers import CLIPImageProcessor, SiglipImageProcessor +from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig, BitsAndBytesConfig, AutoProcessor +import warnings +import os +from aloha_scripts.utils import * +def find_all_linear_names(model, rank0_print, lora_module=None): + cls = torch.nn.Linear + lora_module_names = set() + + multimodal_keywords = ['multi_modal_projector', 'lm_head', 'xattn', 'input_action_proj', 'gt_film', 'gt_action_proj', 'reasoning_action_proj', 'reasoning_film', 'merger'] + if 'vit' not in lora_module: + multimodal_keywords.append("vision_tower") + if 'llm' not in lora_module: + multimodal_keywords.append("language_model") + if 'di_head' not in lora_module: # not lora finetune policy_head + multimodal_keywords.append("policy_head") + else: # lora policy_head + multimodal_keywords.append("x_embedder") + multimodal_keywords.append("cond_obs_emb") + multimodal_keywords.append("norm_after_pool") + + + rank0_print("##" * 20) + + for name, module in model.named_modules(): + if any(mm_keyword in name for mm_keyword in multimodal_keywords): + continue + + if isinstance(module, cls): + lora_module_names.add(name) + + if 'lm_head' in lora_module_names: # needed for 16-bit + lora_module_names.remove('lm_head') + + return list(lora_module_names) + +def load_model(config=None, qwen2_vla_config=None, rank0_print=print, tokenizer=None): + model_args = config['model_args'] + training_args = config['training_args'] + data_args = config['data_args'] + action_args = config['action_head_args'] + + # model_arch = paligemma_config.architectures[0] + if training_args.load_pretrain: # loading pretrained weights + pass + kwargs = {"device_map": "cuda", "torch_dtype": torch.bfloat16} + rank0_print(f"@@@@@@@Loading pretrain weights...@@@@@@@@@@") + assert config['model_args'].model_pretrain is not "", "load pretrain weights need set the model_pretrain in DataArguments!!!!" + # models = load_pretrained_model(config['model_args'].model_pretrain, config['model_args'].model_name_or_path, model_name, False, False) + model_path = config['model_args'].model_pretrain + model_base = config['model_args'].model_name_or_path + path = model_path.split('/')[0:-1] + root_path = '/'.join(path) + # lora_cfg_pretrained = AutoConfig.from_pretrained(root_path) + # config = lora_cfg_pretrained + tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=True) # default use_fast=False + rank0_print(f"{RED}Loading pretrained <<{config['model_args'].model_pretrain}>> from base models...{RESET}") + # model = AutoModelForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=qwen2_vla_config,**kwargs) + if config['training_args'].flash_attn: + model = AutoModelForCausalLM.from_pretrained( + model_base, + config=qwen2_vla_config, + cache_dir=config['training_args'].cache_dir, + trust_remote_code=True, + _fast_init=False, + attn_implementation="flash_attention_2", + ) + else: + model = AutoModelForCausalLM.from_pretrained( + model_base, + config=qwen2_vla_config, + cache_dir=config['training_args'].cache_dir, + trust_remote_code=True, + _fast_init=False, + # attn_implementation="flash_attention_2", + ) + # rank0_print(f'{RED} Only loading lora weights from pretrained model because the stage_1(pretrain) only lora the VLM {RESET}') + + rank0_print(f'Loading pretrained additional <<{model_path}/non_lora_trainables.bin>> weights...') + if os.path.exists(os.path.join(model_path, 'non_lora_trainables.bin')): + non_lora_trainables = torch.load(os.path.join(model_path, 'non_lora_trainables.bin'), map_location='cpu') + else: + raise f"there is no non_lora_trainables.bin in {model_path}" + + non_lora_trainables = load_from_hf(model_path, 'non_lora_trainables.bin') + # todo length of paligemma is different from pythia + non_lora_trainables = {(k[11:] if k.startswith('base_model.') else k): v for k, v in + non_lora_trainables.items()} + if any(k.startswith('model.policy_head.') for k in non_lora_trainables): + non_lora_trainables = {(k[6:] if k.startswith('model.') else k): v for k, v in + non_lora_trainables.items()} + + # 删除lora相关的参数 + keys_to_del = [] + for k, v in non_lora_trainables.items(): + if 'lora' in k: + keys_to_del.append(k) + + # keys_to_del = ['policy_head.final_conv.1.weight', 'policy_head.final_conv.1.bias'] + # todo + # if config['action_head_args'].action_dim == 144: + # keys_to_del = [] + # rank0_print(f"{RED}Deleting some modules to adapt for bimanual setting....{RESET}") + # for name in ['policy_head.combine.weight','policy_head.down_modules.0.0.blocks.0.block.0.weight', 'policy_head.down_modules.0.0.residual_conv.weight', + # 'policy_head.final_conv.1.weight', 'policy_head.final_conv.1.bias']: + # keys_to_del.append(name) + # rank0_print(">>"*30) + # rank0_print(f"Reinitializing weights of followings:{keys_to_del}") + # print(keys_to_del) + # print("#"*40) + # print(pretrain.keys()) + # exit(0) + for key in keys_to_del: + del non_lora_trainables[key] + + model.load_state_dict(non_lora_trainables, strict=False) + + from peft import PeftModel + rank0_print('Loading LoRA weights...') + model = PeftModel.from_pretrained(model, model_path) + rank0_print('Merging LoRA weights...') + model = model.merge_and_unload() + rank0_print('Model is loaded...') + model.to(torch.bfloat16) + # else: + else: + kwargs = {"device_map": "cuda", "torch_dtype": torch.bfloat16} + if config['training_args'].flash_attn: + if 'paligemma' in config['model_args'].model_name_or_path.lower(): + flash_attn = "eager" + else: + flash_attn = "flash_attention_2" + model = AutoModelForCausalLM.from_pretrained( + config['model_args'].model_name_or_path, + config=qwen2_vla_config, + cache_dir=config['training_args'].cache_dir, + trust_remote_code=True, + _fast_init=False, + attn_implementation=flash_attn, + ) + else: + model = AutoModelForCausalLM.from_pretrained( + config['model_args'].model_name_or_path, + config=qwen2_vla_config, + cache_dir=config['training_args'].cache_dir, + trust_remote_code=True, + _fast_init=False, + # attn_implementation="flash_attention_2", + # **kwargs, # specified device map and dtype may cause nan initialize + ) + + if model_args.load_pretrain_dit and not config['training_args'].resume_from_checkpoint: + assert model_args.pretrain_dit_path is not None, "please specify a pretrained dit path when setting load_pretrain_dit==True" + rank0_print(f'{RED}>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Loading pretrained dit weights...<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<{RESET}') + pretrain_dit_weights = torch.load(model_args.pretrain_dit_path, map_location='cpu') + if (not model_args.Using_EMA_Pretrain_DiT) or ("use_constant_1" in model_args.pretrain_dit_path): + rank0_print(f'{RED} << Load Non-Non-Non-EMA weights>>{RESET}') + pretrain_dit_weights = pretrain_dit_weights['nets']['nets'] + else: + rank0_print(f'{RED} << Load EMA weights>>{RESET}') + if 'nets' in pretrain_dit_weights.keys(): + pretrain_dit_weights = pretrain_dit_weights['nets']['ema'] + else: + pretrain_dit_weights = pretrain_dit_weights['ema'] + keys_to_del_dit = [] + pretrain_dit_weights = {k[7:] if k.startswith('policy.') else k: v for k, v in pretrain_dit_weights.items()} + for k in pretrain_dit_weights.keys(): + # if 'noise_pred' not in k: # del weights of vision backbones + # keys_to_del_dit.append(k) + if model_args.external_vision_encoder == "None": + if 'noise_pred' not in k: # del weights of vision backbones + keys_to_del_dit.append(k) + else: + if 'combine' in k or 'film' in k: + keys_to_del_dit.append(k) + if 'cond_obs_emb' in k: + keys_to_del_dit.append(k) + for k in keys_to_del_dit: + del pretrain_dit_weights[k] + pretrain_dit_weights = {k[15:] if k.startswith('noise_pred_net.') else k: v for k, v in pretrain_dit_weights.items()} + + model.policy_head.load_state_dict(pretrain_dit_weights, strict=False) + if model_args.external_vision_encoder != "None": + model.external_vision_encoder_model.load_state_dict(pretrain_dit_weights, strict=False) + + + model.config.use_cache = False + + model_args.freeze_backbone = training_args.freeze_backbone + if model_args.freeze_backbone: + model.requires_grad_(False) + else: + model.requires_grad_(True) + + if 'paligemma' in config['model_args'].model_name_or_path.lower(): + model.vision_tower.requires_grad_(True) # set to true first + model.config.freeze_vision_tower = model_args.freeze_vision_tower = training_args.freeze_vision_tower + if model_args.freeze_vision_tower: + for n, p in model.vision_tower.named_parameters(): + if not 'lora' in n.lower(): + p.requires_grad = False + else: + for p in model.vision_tower.parameters(): + p.requires_grad = True + else: + model.visual.requires_grad_(True) # set to true first + model.config.freeze_vision_tower = model_args.freeze_vision_tower = training_args.freeze_vision_tower + if model_args.freeze_vision_tower: + for n,p in model.visual.named_parameters(): + if not 'lora' in n.lower(): + p.requires_grad = False + else: + for p in model.visual.parameters(): + p.requires_grad = True + + + if training_args.bits in [4, 8]: + from peft import prepare_model_for_kbit_training + model.config.torch_dtype = ( + torch.float32 if training_args.fp16 else (torch.bfloat16 if training_args.bf16 else torch.float32)) + model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=training_args.gradient_checkpointing) + + # TODO: https://huggingface.co/microsoft/phi-2/discussions/31. But in this code, setting gradient_checkpointing=True, it doesn't raise any error + if training_args.gradient_checkpointing: + if hasattr(model, "enable_input_require_grads"): + model.enable_input_require_grads() + else: + def make_inputs_require_grad(module, input, output): + output.requires_grad_(True) + + model.get_input_embeddings().register_forward_hook(make_inputs_require_grad) + + # if training_args.lora_enable and (not training_args.load_pretrain): + if training_args.lora_enable: + from peft import LoraConfig, get_peft_model + lora_config = LoraConfig( + r=training_args.lora_r, + lora_alpha=training_args.lora_alpha, + target_modules=find_all_linear_names(model, rank0_print, training_args.lora_module), + lora_dropout=training_args.lora_dropout, + bias=training_args.lora_bias, + task_type=training_args.lora_task_type, + ) + if training_args.bits == 16: + if training_args.bf16: + model.to(torch.bfloat16) + if training_args.fp16: + model.to(torch.float16) + rank0_print("##" * 20) + + rank0_print("Adding LoRA adapters...") + model = get_peft_model(model, lora_config) # !!!only set lora weights to requires_grad True!!! + rank0_print(model) + model.print_trainable_parameters() + elif training_args.load_pretrain: + rank0_print("Already loaded pretrained weights which is based on lora, skipping LoRA initialize...") + + + model.config.tune_mm_mlp_adapter = model_args.tune_mm_mlp_adapter = training_args.tune_mm_mlp_adapter + + # if not model_args.tune_mm_mlp_adapter: + # for p in model.multi_modal_projector.parameters(): + # p.requires_grad = False + # else: + # for p in model.multi_modal_projector.parameters(): + # p.requires_grad = True + if config['model_args'].with_llm_head and not model_args.freeze_backbone: + try: + model.lm_head.requires_grad_(True) + except Exception as e: + rank0_print(e) + model.language_model.lm_head.requires_grad_(True) + # action head需要训练 + if 'di_head' in training_args.lora_module: + model.policy_head.x_embedder.requires_grad_(True) + model.policy_head.cond_obs_emb.requires_grad_(True) + # model.policy_head.norm_after_pool.requires_grad_(True) + + else: + if not model_args.freeze_policy_head: + model.policy_head.requires_grad_(True) + + if config['model_args'].with_text_fcs: + model.text_hidden_fcs.requires_grad_(True) + if config['model_args'].using_film or config['model_args'].using_channel_cat: + model.input_action_proj.requires_grad_(True) + model.reasoning_action_proj.requires_grad_(True) + if config['model_args'].using_all_reasoning_hidden: + model.gt_action_proj.requires_grad_(True) + model.gt_film.requires_grad_(True) + if config['model_args'].using_film: + model.reasoning_film.requires_grad_(True) + if config['model_args'].using_xattn: + model.xattn.requires_grad_(True) + model.xattn.to(torch.bfloat16) + + if 'paligemma' in config['model_args'].model_name_or_path.lower(): + vision_tower = model.vision_tower + else: + vision_tower = model.visual + + vision_tower.to(dtype=torch.bfloat16 if training_args.bf16 else torch.float16, device=training_args.device) + model.to(dtype=torch.bfloat16 if training_args.bf16 else torch.float16, device=training_args.device) + + + for k, v in model.named_parameters(): + if v.requires_grad: + if 'film' in k or 'action_proj' in k: + rank0_print(f"{RED}{k}{RESET}", v.requires_grad, v.dtype) + else: + rank0_print(k, v.requires_grad, v.dtype) + + compute_dtype = (torch.float16 if training_args.fp16 else (torch.bfloat16 if training_args.bf16 else torch.float32)) + + if training_args.bits in [4, 8]: + model.multi_modal_projector.to(dtype=compute_dtype, device=training_args.device) + + # model.config.mm_use_im_start_end = data_args.mm_use_im_start_end = model_args.mm_use_im_start_end + model.config.non_lora_lr = training_args.non_lora_lr + + + if training_args.bits in [4, 8]: + from peft.tuners.lora import LoraLayer + for name, module in model.named_modules(): + if isinstance(module, LoraLayer): + if training_args.bf16: + module = module.to(torch.bfloat16) + if 'norm' in name: + module = module.to(torch.float32) + if 'lm_head' in name or 'embed_tokens' in name: + if hasattr(module, 'weight'): + if training_args.bf16 and module.weight.dtype == torch.float32: + module = module.to(torch.bfloat16) + + rank0_print("!"*100) + lora_para = sum(p.numel() for n, p in model.named_parameters() if (p.requires_grad and 'lora' in n)) + all_para = sum(p.numel() for n, p in model.named_parameters()) + train_para = sum(p.numel() for n, p in model.named_parameters() if p.requires_grad) + rank0_print(f"{RED}Lora parameters/trainalbe parameters/all parameters:{lora_para/1000000}M/{train_para/1000000}M/{(all_para-lora_para)/1000000}M{RESET}") + # print(sum(p.numel() for n, p in model.embed_out.named_parameters() if p.requires_grad)/1000000) + + return model, data_args + +def maybe_zero_3(param, ignore_status=False, name=None): + from deepspeed import zero + from deepspeed.runtime.zero.partition_parameters import ZeroParamStatus + if hasattr(param, "ds_id"): + if param.ds_status == ZeroParamStatus.NOT_AVAILABLE: + if not ignore_status: + logging.warning(f"{name}: param.ds_status != ZeroParamStatus.NOT_AVAILABLE: {param.ds_status}") + with zero.GatheredParameters([param]): + param = param.data.detach().cpu().clone() + else: + param = param.detach().cpu().clone() + return param + + +# Borrowed from peft.utils.get_peft_model_state_dict +def get_peft_state_maybe_zero_3(named_params, bias): + if bias == "none": + to_return = {k: t for k, t in named_params if "lora_" in k} + elif bias == "all": + to_return = {k: t for k, t in named_params if "lora_" in k or "bias" in k} + elif bias == "lora_only": + to_return = {} + maybe_lora_bias = {} + lora_bias_names = set() + for k, t in named_params: + if "lora_" in k: + to_return[k] = t + bias_name = k.split("lora_")[0] + "bias" + lora_bias_names.add(bias_name) + elif "bias" in k: + maybe_lora_bias[k] = t + for k, t in maybe_lora_bias: + if bias_name in lora_bias_names: + to_return[bias_name] = t + else: + raise NotImplementedError + to_return = {k: maybe_zero_3(v, ignore_status=True) for k, v in to_return.items()} + return to_return + + +def get_peft_state_non_lora_maybe_zero_3(named_params, require_grad_only=True): + to_return = {k: t for k, t in named_params if "lora_" not in k} + if require_grad_only: + to_return = {k: t for k, t in to_return.items() if t.requires_grad} + to_return = {k: maybe_zero_3(v, ignore_status=True).cpu() for k, v in to_return.items()} + return to_return + +def safe_save_model_for_hf_trainer(trainer: transformers.Trainer, + output_dir: str): + """Collects the state dict and dump to disk.""" + + if trainer.deepspeed: + torch.cuda.synchronize() + trainer.save_model(output_dir) + return + + state_dict = trainer.model.state_dict() + if trainer.args.should_save: + cpu_state_dict = { + key: value.cpu() + for key, value in state_dict.items() + } + del state_dict + trainer._save(output_dir, state_dict=cpu_state_dict) # noqa + +def load_merge_lora_weights(model_path=None, model_base=None, kwargs=None, pretrain_dit_path=None): + path = model_path.split('/')[0:-1] + if 'checkpoint' in path[-1]: + path = path[:-1] + root_path = '/'.join(path) + lora_cfg_pretrained = AutoConfig.from_pretrained(root_path) + # config = lora_cfg_pretrained + tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=True) # default use_fast=False + print('Loading QWen2-VLA from base model...') + model = AutoModelForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, + config=lora_cfg_pretrained, **kwargs) + + print('Loading additional QWen2-VLA weights expecially non-lora part(diffusion head)...') + if os.path.exists(os.path.join(model_path, 'ema_adapter')): + non_lora_trainables = torch.load(os.path.join(model_path, 'ema_adapter', 'ema_non_lora_trainables.bin'), ) + elif os.path.exists(os.path.join(model_path, 'non_lora_trainables.bin')): + non_lora_trainables = torch.load(os.path.join(model_path, 'non_lora_trainables.bin'),) + else: + # this is probably from HF Hub + from huggingface_hub import hf_hub_download + def load_from_hf(repo_id, filename, subfolder=None): + cache_file = hf_hub_download( + repo_id=repo_id, + filename=filename, + subfolder=subfolder) + return torch.load(cache_file, map_location='cpu') + + non_lora_trainables = load_from_hf(model_path, 'non_lora_trainables.bin') + non_lora_trainables = {(k[11:] if k.startswith('base_model.') else k): v for k, v in + non_lora_trainables.items()} + if any(k.startswith('model.policy_head.') for k in non_lora_trainables): + non_lora_trainables = {(k[6:] if k.startswith('model.') else k): v for k, v in + non_lora_trainables.items()} + + # 删除lora相关的参数 + keys_to_del = [] + for k, v in non_lora_trainables.items(): + if 'lora' in k: + keys_to_del.append(k) + for key in keys_to_del: + del non_lora_trainables[key] + + model.load_state_dict(non_lora_trainables, strict=False) + + if pretrain_dit_path is not None: + print( + f'{RED}>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Loading pretrained dit weights...<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<{RESET}') + pretrain_dit_weights = torch.load(pretrain_dit_path, map_location='cpu')['nets']['ema'] + keys_to_del_dit = [] + pretrain_dit_weights = {k[7:] if k.startswith('policy.') else k: v for k, v in pretrain_dit_weights.items()} + for k in pretrain_dit_weights.keys(): + if 'noise_pred' not in k: + keys_to_del_dit.append(k) + if 'cond_obs_emb' in k: + keys_to_del_dit.append(k) + + for k in keys_to_del_dit: + del pretrain_dit_weights[k] + pretrain_dit_weights = {k[15:] if k.startswith('noise_pred_net.') else k: v for k, v in + pretrain_dit_weights.items()} + + model.policy_head.load_state_dict(pretrain_dit_weights, strict=False) + + from peft import PeftModel + if os.path.exists(os.path.join(model_path, "adapter_model.safetensors")) and os.path.exists(os.path.join(model_path, 'ema_adapter')): + print('Loading EMA LoRA weights...') + model = PeftModel.from_pretrained(model, os.path.join(model_path, 'ema_adapter')) + print('Merging EMA LoRA weights...') + model = model.merge_and_unload() + print('Model is loaded...') + elif os.path.exists(os.path.join(model_path, "adapter_model.safetensors")): + print('Loading LoRA weights...') + model = PeftModel.from_pretrained(model, model_path) + print('Merging LoRA weights...') + model = model.merge_and_unload() + print('Model is loaded...') + else: + print("There is no lora...") + return model, tokenizer + +def load_model_for_eval(model_path, model_base, load_8bit=False, load_4bit=False, device_map="cuda:0", policy_config=None): + kwargs = {"device_map": device_map} + if load_8bit: + kwargs['load_in_8bit'] = True + elif load_4bit: + kwargs['load_in_4bit'] = True + kwargs['quantization_config'] = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_compute_dtype=torch.float16, + bnb_4bit_use_double_quant=True, + bnb_4bit_quant_type='nf4' + ) + else: + kwargs['torch_dtype'] = torch.bfloat16 + if policy_config['save_model']: + kwargs['torch_dtype'] = torch.bfloat16 + + if model_base is not None and '72B' in model_base: + kwargs = { + "device_map":"cpu", + "max_memory":{0:"45GiB", 1:"45GiB", "cpu":"80GiB"}, + "offload_folder": "/home/eai/wjj/qwen2_vla/offload", + "offload_state_dict": True, + } + with open(os.path.join(model_base, 'device_map.json'), 'r') as f: + device_map = json.load(f) + kwargs['device_map'] = device_map + + # if os.path.exists(os.path.join(model_path, 'merge_weights')) and len(os.listdir(os.path.join(model_path, 'merge_weights'))) > 1: + # kwargs['torch_dtype'] = torch.bfloat16 + # model = AutoModelForCausalLM.from_pretrained(os.path.join(model_path, 'merge_weights'), low_cpu_mem_usage=True, + # **kwargs) + # tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + # model = model.to(torch.bfloat16) + if False: + pass + elif 'qwen2' in model_path.lower() or 'paligemma' in model_path.lower(): + # Load LLaVA-Phi model + if 'lora' in model_path.lower() and model_base is None: + warnings.warn( + 'There is `lora` in model name but no `model_base` is provided. If you are loading a LoRA model, please provide the `model_base` argument.') + if 'lora' in model_path.lower() and model_base is not None: + if policy_config['pretrain_path'] is not None: + ps = model_path.split('/') + # parent_model_path = '/'.join(ps[:-1]) + if not os.path.exists(os.path.join(policy_config['pretrain_path'], 'pretrain_merge_weights')): + print("merging pretrained weights.......") + model, tokenizer = load_merge_lora_weights(model_path=policy_config['pretrain_path'], model_base=model_base, kwargs=kwargs) + + os.makedirs(os.path.join(policy_config['pretrain_path'], 'pretrain_merge_weights'), exist_ok=True) + model.save_pretrained( + os.path.join(policy_config['pretrain_path'], 'pretrain_merge_weights')) + tokenizer.save_pretrained(os.path.join(policy_config['pretrain_path'], 'pretrain_merge_weights')) + # multi_modal_processor = AutoProcessor.from_pretrained(parent_model_path, use_fast=False) + # multi_modal_processor.save_pretrained(os.path.join(parent_model_path, 'pretrain_merge_weights')) + print("loading pretrained weights as base model.......") + model, tokenizer = load_merge_lora_weights(model_path=model_path, model_base=os.path.join(policy_config['pretrain_path'], 'pretrain_merge_weights'), kwargs=kwargs) + + else: + model, tokenizer = load_merge_lora_weights(model_path=model_path, model_base=model_base, kwargs=kwargs, pretrain_dit_path=policy_config['pretrain_dit_path']) + + if policy_config['save_model']: + print(f"#####################################Saving merged weights of model in {kwargs['torch_dtype']}.#####################################") + os.makedirs(os.path.join(model_path, 'merge_weights'), exist_ok=True) + model.save_pretrained( + os.path.join(model_path, 'merge_weights')) + tokenizer.save_pretrained(os.path.join(model_path, 'merge_weights')) + skip_params = [ + "input_action_proj", + "policy_head", + "reasoning_action_proj", + "reasoning_film", + ] + head_param = {} + for k,v in model.named_parameters(): + if any(skip_param in k.lower() for skip_param in skip_params): + head_param[k] = v + torch.save(head_param, os.path.join(model_path, 'merge_weights/head_params.bin')) + multi_modal_processor = AutoProcessor.from_pretrained(model_path, use_fast=False) + multi_modal_processor.save_pretrained(os.path.join(model_path, 'merge_weights')) + exit(0) + + # model = model.to(torch.bfloat16) + elif model_base is not None: + # this may be mm projector only + print(f'Loading {model_base.split("/")[-1]} from base model...') + tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False) + cfg_pretrained = AutoConfig.from_pretrained(model_path) + model = AutoModelForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=cfg_pretrained, + **kwargs) + + mm_projector_weights = torch.load(os.path.join(model_path, 'mm_projector.bin'), map_location='cpu') + mm_projector_weights = {k: v.to(torch.float16) for k, v in mm_projector_weights.items()} + model.load_state_dict(mm_projector_weights, strict=False) + else: + print(f"load {model_path.split('/')[-1]}!!!") + config = AutoConfig.from_pretrained(model_path, trust_remote_code=True) + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True) + model = AutoModelForCausalLM.from_pretrained( + model_path, + config=config, + use_safetensors=True, + **kwargs) + else: + # Load language model + if model_base is not None: + # PEFT model + from peft import PeftModel + tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False) + model = AutoModelForCausalLM.from_pretrained(model_base, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True, + device_map="auto") + print(f"Loading LoRA weights from {model_path}") + model = PeftModel.from_pretrained(model, model_path) + print(f"Merging weights") + model = model.merge_and_unload() + print('Convert to FP16...') + model.to(torch.bfloat16) + else: + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + model = AutoModelForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, **kwargs) + + print("aaaa") + # image_processor = AutoImageProcessor.from_pretrained(model_path, use_fast=False) + # multi_modal_processor = Qwen2VLProcessor.from_pretrained(model_path, use_fast=False) + # multi_modal_processor.image_processor = image_processor + multi_modal_processor = AutoProcessor.from_pretrained(model_path, use_fast=False) + if hasattr(model.config, "max_sequence_length"): + context_len = model.config.max_sequence_length + else: + context_len = 2048 + model.to(device="cuda") + print(kwargs) + # print(model) + return tokenizer, model, multi_modal_processor, context_len + diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/models/__init__.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/models/__init__.py new file mode 100644 index 00000000..65c9710e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/models/__init__.py @@ -0,0 +1,73 @@ +# Copyright 2024 The Qwen Team and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING + +from transformers.utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available + + +_import_structure = { + "configuration_qwen2_vl": ["Qwen2VLConfig"], + "processing_qwen2_vl": ["Qwen2VLProcessor"], +} + + +try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + pass +else: + _import_structure["modeling_qwen2_vl"] = [ + "Qwen2VLForConditionalGeneration", + "Qwen2VLModel", + "Qwen2VLPreTrainedModel", + ] + +try: + if not is_vision_available(): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + pass +else: + _import_structure["image_processing_qwen2_vl"] = ["Qwen2VLImageProcessor"] + + +if TYPE_CHECKING: + from .configuration_dex_vla import Qwen2VLConfig + + try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + pass + else: + from .modeling_dex_vla import ( + Qwen2VLForConditionalGeneration, + Qwen2VLModel, + Qwen2VLPreTrainedModel, + ) + + try: + if not is_vision_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + pass + else: + from dex_vla.utils.image_processing_qwen2_vla import Qwen2VLImageProcessor + + +else: + import sys + + sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/models/__pycache__/__init__.cpython-310.pyc b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/models/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 00000000..ca13bfab Binary files /dev/null and b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/models/__pycache__/__init__.cpython-310.pyc differ diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/models/__pycache__/configuration_dex_vla.cpython-310.pyc b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/models/__pycache__/configuration_dex_vla.cpython-310.pyc new file mode 100644 index 00000000..5d7d091c Binary files /dev/null and b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/models/__pycache__/configuration_dex_vla.cpython-310.pyc differ diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/models/__pycache__/modeling_dex_vla.cpython-310.pyc b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/models/__pycache__/modeling_dex_vla.cpython-310.pyc new file mode 100644 index 00000000..8e9494e8 Binary files /dev/null and b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/models/__pycache__/modeling_dex_vla.cpython-310.pyc differ diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/models/__pycache__/qwen2_vl_modules.cpython-310.pyc b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/models/__pycache__/qwen2_vl_modules.cpython-310.pyc new file mode 100644 index 00000000..deaa1b68 Binary files /dev/null and b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/models/__pycache__/qwen2_vl_modules.cpython-310.pyc differ diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/models/configuration_dex_vla.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/models/configuration_dex_vla.py new file mode 100644 index 00000000..14ad5eb9 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/models/configuration_dex_vla.py @@ -0,0 +1,270 @@ +# coding=utf-8 +# Copyright 2024 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Qwen2VL model configuration""" + +import os +from typing import Union + +from transformers.configuration_utils import PretrainedConfig +from transformers.modeling_rope_utils import rope_config_validation +from transformers.utils import logging +from transformers import AutoModel, AutoConfig + +logger = logging.get_logger(__name__) + + +class Qwen2VLAVisionConfig(PretrainedConfig): + model_type = "dex_vla" + + def __init__( + self, + depth=32, + embed_dim=1280, + hidden_size=3584, + hidden_act="quick_gelu", + mlp_ratio=4, + num_heads=16, + in_channels=3, + patch_size=14, + spatial_merge_size=2, + temporal_patch_size=2, + camera_names=None, + **kwargs, + ): + super().__init__(**kwargs) + + self.depth = depth + self.embed_dim = embed_dim + self.hidden_size = hidden_size + self.hidden_act = hidden_act + self.mlp_ratio = mlp_ratio + self.num_heads = num_heads + self.in_channels = in_channels + self.patch_size = patch_size + self.spatial_merge_size = spatial_merge_size + self.temporal_patch_size = temporal_patch_size + self.camera_names = camera_names + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) + + if config_dict.get("model_type") == "qwen2_vl": + config_dict = config_dict["vision_config"] + + if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type: + logger.warning( + f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " + f"{cls.model_type}. This is not supported for all configurations of models and can yield errors." + ) + + return cls.from_dict(config_dict, **kwargs) + + +class DexVLAConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`Qwen2VLModel`]. It is used to instantiate a + Qwen2-VL model according to the specified arguments, defining the model architecture. Instantiating a configuration + with the defaults will yield a similar configuration to that of + Qwen2-VL-7B-Instruct [Qwen/Qwen2-VL-7B-Instruct](https://huggingface.co/Qwen/Qwen2-VL-7B-Instruct). + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 152064): + Vocabulary size of the Qwen2VL model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`Qwen2VLModel`] + hidden_size (`int`, *optional*, defaults to 8192): + Dimension of the hidden representations. + intermediate_size (`int`, *optional*, defaults to 29568): + Dimension of the MLP representations. + num_hidden_layers (`int`, *optional*, defaults to 80): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 64): + Number of attention heads for each attention layer in the Transformer encoder. + num_key_value_heads (`int`, *optional*, defaults to 8): + This is the number of key_value heads that should be used to implement Grouped Query Attention. If + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if + `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When + converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed + by meanpooling all the original heads within that group. For more details checkout [this + paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `32`. + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the decoder. + max_position_embeddings (`int`, *optional*, defaults to 32768): + The maximum sequence length that this model might ever be used with. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + rms_norm_eps (`float`, *optional*, defaults to 1e-05): + The epsilon used by the rms normalization layers. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. + tie_word_embeddings (`bool`, *optional*, defaults to `False`): + Whether the model's input and output word embeddings should be tied. + rope_theta (`float`, *optional*, defaults to 1000000.0): + The base period of the RoPE embeddings. + use_sliding_window (`bool`, *optional*, defaults to `False`): + Whether to use sliding window attention. + sliding_window (`int`, *optional*, defaults to 4096): + Sliding window attention (SWA) window size. If not specified, will default to `4096`. + max_window_layers (`int`, *optional*, defaults to 80): + The number of layers that use SWA (Sliding Window Attention). The bottom layers use SWA while the top use full attention. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + vision_config (`Dict`, *optional*): + The config for the visual encoder initialization. + rope_scaling (`Dict`, *optional*): + Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type + and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value + accordingly. + Expected contents: + `rope_type` (`str`): + The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope', + 'llama3'], with 'default' being the original RoPE implementation. + `factor` (`float`, *optional*): + Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In + most scaling types, a `factor` of x will enable the model to handle sequences of length x * + original maximum pre-trained length. + `original_max_position_embeddings` (`int`, *optional*): + Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during + pretraining. + `attention_factor` (`float`, *optional*): + Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention + computation. If unspecified, it defaults to value recommended by the implementation, using the + `factor` field to infer the suggested value. + `beta_fast` (`float`, *optional*): + Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear + ramp function. If unspecified, it defaults to 32. + `beta_slow` (`float`, *optional*): + Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear + ramp function. If unspecified, it defaults to 1. + `short_factor` (`List[float]`, *optional*): + Only used with 'longrope'. The scaling factor to be applied to short contexts (< + `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden + size divided by the number of attention heads divided by 2 + `long_factor` (`List[float]`, *optional*): + Only used with 'longrope'. The scaling factor to be applied to long contexts (< + `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden + size divided by the number of attention heads divided by 2 + `low_freq_factor` (`float`, *optional*): + Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE + `high_freq_factor` (`float`, *optional*): + Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE + + ```python + >>> from transformers import Qwen2VLForConditionalGeneration, Qwen2VLConfig + + >>> # Initializing a Qwen2VL style configuration + >>> configuration = Qwen2VLConfig() + + >>> # Initializing a model from the Qwen2-VL-7B style configuration + >>> model = Qwen2VLForConditionalGeneration(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "dex_vla" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + vocab_size=152064, + hidden_size=8192, + intermediate_size=29568, + num_hidden_layers=80, + num_attention_heads=64, + num_key_value_heads=8, + hidden_act="silu", + max_position_embeddings=32768, + initializer_range=0.02, + rms_norm_eps=1e-05, + use_cache=True, + tie_word_embeddings=False, + rope_theta=1000000.0, + use_sliding_window=False, + sliding_window=4096, + max_window_layers=80, + attention_dropout=0.0, + vision_config=None, + rope_scaling=None, + # For loading policy head + policy_head_type='dit_diffusion_policy', # dit_diffusion_policy + using_state=False, + **kwargs, + ): + if isinstance(vision_config, dict): + self.vision_config = Qwen2VLAVisionConfig(**vision_config) + elif vision_config is None: + self.vision_config = Qwen2VLAVisionConfig() + + self.using_state = using_state + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.use_sliding_window = use_sliding_window + self.sliding_window = sliding_window + self.max_window_layers = max_window_layers + + # for loading policy head + self.policy_head_type = policy_head_type + # if policy_head_type == 'dit_diffusion_policy': + # # self.policy_head_size = kwargs.get("policy_head_size", "none") + # self.policy_head_size = policy_head_size + # # self.policy_head_config = register_configuration_class(self.policy_head_type, model_size=policy_head_size) + # self.policy_head_config = AutoConfig.for_model(model_type=self.policy_head_type, + # model_size=self.policy_head_size, + # global_cond_dim=hidden_size, action_dim=action_dim, + # state_dim=state_dim) + # elif policy_head_type == 'unet_diffusion_policy': + # self.policy_head_config = AutoConfig.for_model(model_type=self.policy_head_type, + # global_cond_dim=hidden_size, action_dim=action_dim, + # state_dim=state_dim) + # for backward compatibility + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + + self.num_key_value_heads = num_key_value_heads + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.rope_theta = rope_theta + self.attention_dropout = attention_dropout + self.rope_scaling = rope_scaling + + # Validate the correctness of rotary position embeddings parameters + # BC: if there is a 'type' field, move it to 'rope_type'. + # and change type from 'mrope' to 'default' because `mrope` does defeault RoPE calculations + # one can set it to "linear"/"dynamic" etc. to have scaled RoPE + # TODO: @raushan update config in the hub + if self.rope_scaling is not None and "type" in self.rope_scaling: + if self.rope_scaling["type"] == "mrope": + self.rope_scaling["type"] = "default" + self.rope_scaling["rope_type"] = self.rope_scaling["type"] + rope_config_validation(self, ignore_keys={"mrope_section"}) + + super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs) + +from transformers import AutoConfig +AutoConfig.register("dex_vla", DexVLAConfig) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/models/modeling_dex_vla.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/models/modeling_dex_vla.py new file mode 100644 index 00000000..cb96cf34 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/models/modeling_dex_vla.py @@ -0,0 +1,777 @@ +from .qwen2_vl_modules import * +from .qwen2_vl_modules import _CONFIG_FOR_DOC +from dex_vla.utils.fusion_modules import * +class DexVLA(Qwen2VLPreTrainedModel, GenerationMixin): + _tied_weights_keys = ["lm_head.weight"] + + def __init__(self, config): + super().__init__(config) + self.visual = Qwen2VisionTransformerPretrainedModel._from_config( + config.vision_config, attn_implementation=config._attn_implementation + ) + self.model = Qwen2VLModel(config) + self.vocab_size = config.vocab_size + + self.padding_side = "left" # set it to left by default, user can use setter to change padding_sides + self.with_llm_head = config.with_llm_head + # self.with_external_vit = config.with_external_vit + self.with_text_fcs = config.with_text_fcs + self.only_using_input_embeddings = config.only_using_input_embeddings + self.using_film = config.using_film + + self.using_first_layer_hidden_states = getattr(config, "using_first_layer_hidden_states", False) + self.llm_loss_weight = config.llm_loss_weight + self.using_state = getattr(config, "using_state", False) + + self.external_vision_encoder = getattr(config, "external_vision_encoder", "None") + + if isinstance(config.policy_head_config, dict): + config.policy_head_config = AutoConfig.for_model(**config.policy_head_config) + self.policy_head = AutoModel.from_config(config=config.policy_head_config) + + # if self.with_llm_head: + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + if self.using_state: + self.state_proj = nn.Linear(config.policy_head_config.state_dim, config.hidden_size) + + # Initialize weights and apply final processing + self.post_init() + if config.policy_head_config.model_type == "dit_diffusion_policy": + self.policy_head.initialize_weights() + self.input_action_proj = ActionProjector(config.hidden_size, config.hidden_size) + self.reasoning_action_proj = ActionProjector(config.hidden_size, config.hidden_size) + + if self.using_film: + self.reasoning_film = FiLM(feature_dim=config.hidden_size, condition_dim=config.hidden_size) + + if self.external_vision_encoder == 'resnet': + from ..external_vision_encoder.resnet_vision_encoder import ResNetEncoder + self.external_vision_encoder_model = ResNetEncoder() + pass + # elif self.using_xattn: + # # xattn_config = { + # # hidden_size: + # # } + # xattn_config = SimpleNamespace( + # hidden_size=config.hidden_size, + # num_attention_heads=config.num_attention_heads, + # encoder_width=config.hidden_size, + # attention_probs_dropout_prob=0.2, + # hidden_dropout_prob=0.2, + # layer_norm_eps=1e-12, + # ) + # self.xattn = BertAttention(xattn_config) + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + def set_decoder(self, decoder): + self.model = decoder + + def get_decoder(self): + return self.model + + def get_rope_index( + self, + input_ids: torch.LongTensor, + image_grid_thw: Optional[torch.LongTensor] = None, + video_grid_thw: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Calculate the 3D rope index based on image and video's temporal, height and width in LLM. + + Explanation: + Each embedding sequence contains vision embedding and text embedding or just contains text embedding. + + For pure text embedding sequence, the rotary position embedding has no difference with mordern LLMs. + Examples: + input_ids: [T T T T T], here T is for text. + temporal position_ids: [0, 1, 2, 3, 4] + height position_ids: [0, 1, 2, 3, 4] + width position_ids: [0, 1, 2, 3, 4] + + For vision and text embedding sequence, we calculate 3D rotary position embedding for vision part + and 1D rotary position embeddin for text part. + Examples: + Assume we have a video input with 3 temporal patches, 2 height patches and 2 width patches. + input_ids: [V V V V V V V V V V V V T T T T T], here V is for vision. + vision temporal position_ids: [0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2] + vision height position_ids: [0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1] + vision width position_ids: [0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1] + text temporal position_ids: [3, 4, 5, 6, 7] + text height position_ids: [3, 4, 5, 6, 7] + text width position_ids: [3, 4, 5, 6, 7] + Here we calculate the text start position_ids as the max vision position_ids plus 1. + + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + Returns: + position_ids (`torch.LongTensor` of shape `(3, batch_size, sequence_length)`) + mrope_position_deltas (`torch.Tensor` of shape `(batch_size)`) + """ + spatial_merge_size = self.config.vision_config.spatial_merge_size + image_token_id = self.config.image_token_id + video_token_id = self.config.video_token_id + vision_start_token_id = self.config.vision_start_token_id + mrope_position_deltas = [] + if image_grid_thw is not None or video_grid_thw is not None: + total_input_ids = input_ids + position_ids = torch.ones( + 3, input_ids.shape[0], input_ids.shape[1], dtype=input_ids.dtype, device=input_ids.device + ) + image_index, video_index = 0, 0 + for i, input_ids in enumerate(total_input_ids): + if attention_mask is not None: + input_ids = input_ids[attention_mask[i] == 1] + image_nums, video_nums = 0, 0 + vision_start_indices = torch.argwhere(input_ids == vision_start_token_id).squeeze(1) + vision_tokens = input_ids[vision_start_indices + 1] + image_nums = (vision_tokens == image_token_id).sum() + video_nums = (vision_tokens == video_token_id).sum() + input_tokens = input_ids.tolist() + llm_pos_ids_list: list = [] + st = 0 + remain_images, remain_videos = image_nums, video_nums + for _ in range(image_nums + video_nums): + if image_token_id in input_tokens and remain_images > 0: + ed_image = input_tokens.index(image_token_id, st) + else: + ed_image = len(input_tokens) + 1 + if video_token_id in input_tokens and remain_videos > 0: + ed_video = input_tokens.index(video_token_id, st) + else: + ed_video = len(input_tokens) + 1 + if ed_image < ed_video: + t, h, w = ( + image_grid_thw[image_index][0], + image_grid_thw[image_index][1], + image_grid_thw[image_index][2], + ) + image_index += 1 + remain_images -= 1 + ed = ed_image + else: + t, h, w = ( + video_grid_thw[video_index][0], + video_grid_thw[video_index][1], + video_grid_thw[video_index][2], + ) + video_index += 1 + remain_videos -= 1 + ed = ed_video + llm_grid_t, llm_grid_h, llm_grid_w = ( + t.item(), + h.item() // spatial_merge_size, + w.item() // spatial_merge_size, + ) + text_len = ed - st + + st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0 + llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx) + + t_index = torch.arange(llm_grid_t).view(-1, 1).expand(-1, llm_grid_h * llm_grid_w).flatten() + h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(llm_grid_t, -1, llm_grid_w).flatten() + w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(llm_grid_t, llm_grid_h, -1).flatten() + llm_pos_ids_list.append(torch.stack([t_index, h_index, w_index]) + text_len + st_idx) + st = ed + llm_grid_t * llm_grid_h * llm_grid_w + + if st < len(input_tokens): + st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0 + text_len = len(input_tokens) - st + llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx) + + llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1) + position_ids[..., i, attention_mask[i] == 1] = llm_positions.to(position_ids.device) + mrope_position_deltas.append(llm_positions.max() + 1 - len(total_input_ids[i])) + mrope_position_deltas = torch.tensor(mrope_position_deltas, device=input_ids.device).unsqueeze(1) + return position_ids, mrope_position_deltas + else: + if attention_mask is not None: + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + position_ids = position_ids.unsqueeze(0).expand(3, -1, -1).to(input_ids.device) + max_position_ids = position_ids.max(0, keepdim=False)[0].max(-1, keepdim=True)[0] + mrope_position_deltas = max_position_ids + 1 - attention_mask.shape[-1] + else: + position_ids = ( + torch.arange(input_ids.shape[1], device=input_ids.device) + .view(1, 1, -1) + .expand(3, input_ids.shape[0], -1) + ) + mrope_position_deltas = torch.zeros( + [input_ids.shape[0], 1], + device=input_ids.device, + dtype=input_ids.dtype, + ) + + return position_ids, mrope_position_deltas + + def _update_model_kwargs_for_generation( + self, + outputs: ModelOutput, + model_kwargs: Dict[str, Any], + is_encoder_decoder: bool = False, + num_new_tokens: int = 1, + ) -> Dict[str, Any]: + model_kwargs = super()._update_model_kwargs_for_generation( + outputs=outputs, + model_kwargs=model_kwargs, + is_encoder_decoder=is_encoder_decoder, + num_new_tokens=num_new_tokens, + ) + + if getattr(outputs, "rope_deltas", None) is not None: + model_kwargs["rope_deltas"] = outputs.rope_deltas + + return model_kwargs + + @add_start_docstrings_to_model_forward(QWEN2_VL_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=Qwen2VLCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + pixel_values: Optional[torch.Tensor] = None, + pixel_values_videos: Optional[torch.FloatTensor] = None, + image_grid_thw: Optional[torch.LongTensor] = None, + video_grid_thw: Optional[torch.LongTensor] = None, + rope_deltas: Optional[torch.LongTensor] = None, + actions: Optional[torch.LongTensor] = None, + states: Optional[torch.FloatTensor] = None, + is_pad: bool = False, + is_eval: bool = False, + tinyvla: bool = False, + raw_images: Optional[torch.Tensor] = None, + ) -> Union[Tuple, Qwen2VLCausalLMOutputWithPast]: + r""" + Args: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Returns: + + Example: + + ```python + >>> from PIL import Image + >>> import requests + >>> from transformers import AutoProcessor, Qwen2VLForConditionalGeneration + + >>> model = Qwen2VLForConditionalGeneration.from_pretrained("Qwen/Qwen2-VL-7B-Instruct") + >>> processor = AutoProcessor.from_pretrained("Qwen/Qwen2-VL-7B-Instruct") + + >>> messages = [ + { + "role": "user", + "content": [ + {"type": "image"}, + {"type": "text", "text": "What is shown in this image?"}, + ], + }, + ] + >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) + >>> inputs = processor(text=[text], images=[image], vision_infos=[vision_infos]) + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "The image shows a street scene with a red stop sign in the foreground. In the background, there is a large red gate with Chinese characters ..." + ```""" + # print("@@"*40) + # for each in [attention_mask, labels, pixel_values, pixel_values_videos, image_grid_thw, actions, states]: + # try: + # print(each.dtype) + # except: + # print('None') + # print(self.model.dtype) + # exit(0) + # print(actions.shape) + self.computed_type = torch.bfloat16 + input_ids=input_ids.to("cuda") + attention_mask=attention_mask.to("cuda") + if not is_eval: + labels = labels.to("cuda") + actions = actions.to(dtype=self.computed_type, device='cuda') + states = states.to(dtype=self.computed_type, device='cuda') + + if self.using_state: + attn_mask_state_embedding = torch.ones((attention_mask.shape[0], 1), dtype=torch.bool, device=attention_mask.device) + attention_mask = torch.cat([attention_mask, attn_mask_state_embedding], dim=-1) + + state_id = torch.ones((input_ids.shape[0], 1), dtype=input_ids.dtype, device=input_ids.device) + input_ids = torch.cat([input_ids, state_id], dim=-1) + # start_idxs = torch.where(labels != -100) + labels = torch.cat([-100 * torch.ones((labels.shape[0], 1), dtype=labels.dtype, device=labels.device), labels], dim=1) + + mask = (input_ids == 151653).int() # shape: [4, 100] + start_idxs = torch.argmax(torch.flip(mask, dims=[1]), dim=1) # shape: [4, 100] + start_idxs = mask.shape[1] - 1 - start_idxs + position_ids, rope_deltas = self.get_rope_index( + input_ids, image_grid_thw, video_grid_thw, attention_mask + ) + else: + if self.using_state: + mask = (input_ids == 151653).int() # shape: [4, 100] + start_idxs = torch.argmax(torch.flip(mask, dims=[1]), dim=1) # shape: [4, 100] + start_idxs = mask.shape[1] - 1 - start_idxs + + if pixel_values is not None: + pixel_values = pixel_values.to(dtype=self.computed_type, device='cuda') + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if inputs_embeds is None: + inputs_embeds = self.model.embed_tokens(input_ids) + ground_truth_reasoning_embed = inputs_embeds + if pixel_values is not None: + pixel_values = pixel_values.type(self.visual.get_dtype()) + image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) + n_image_tokens = (input_ids == self.config.image_token_id).sum().item() + n_image_features = image_embeds.shape[0] + if n_image_tokens != n_image_features: + raise ValueError( + f"Image features and image tokens do not match: tokens: {n_image_tokens}, features {n_image_features}" + ) + image_mask = ( + (input_ids == self.config.image_token_id) + .unsqueeze(-1) + .expand_as(inputs_embeds) + .to(inputs_embeds.device) + ) + image_embeds = image_embeds.to(inputs_embeds.device, inputs_embeds.dtype) + inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) + + if pixel_values_videos is not None: + pixel_values_videos = pixel_values_videos.type(self.visual.get_dtype()) + video_embeds = self.visual(pixel_values_videos, grid_thw=video_grid_thw) + n_video_tokens = (input_ids == self.config.video_token_id).sum().item() + n_video_features = video_embeds.shape[0] + if n_video_tokens != n_video_features: + raise ValueError( + f"Video features and video tokens do not match: tokens: {n_video_tokens}, features {n_video_features}" + ) + video_mask = ( + (input_ids == self.config.video_token_id) + .unsqueeze(-1) + .expand_as(inputs_embeds) + .to(inputs_embeds.device) + ) + video_embeds = video_embeds.to(inputs_embeds.device, inputs_embeds.dtype) + inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds) + + if attention_mask is not None: + attention_mask = attention_mask.to(inputs_embeds.device) + + if self.using_state and inputs_embeds.shape[1] > 1: + state_embedding = self.state_proj(states).unsqueeze(1) + temp = [] + for id, each in enumerate(inputs_embeds): + s_id = start_idxs[id] + 1 + t = torch.cat([each[:s_id], state_embedding[id], each[s_id:-1]], dim=0) + temp.append(t) + + inputs_embeds = torch.stack(temp, dim=0) + + outputs = self.model( + input_ids=None, + position_ids=position_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=True, + return_dict=return_dict, + ) + hidden_states = outputs[0] + if tinyvla: + return hidden_states + if self.with_llm_head: + logits = self.lm_head(hidden_states) + logits = logits.float() + else: + logits = None + self.llm_head = None + + llm_loss = None + if labels is not None and self.with_llm_head: + # Shift so that tokens < n predict n + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + # Flatten the tokens + loss_fct = CrossEntropyLoss() + shift_logits = shift_logits.view(-1, self.config.vocab_size) + shift_labels = shift_labels.view(-1) + # Enable model parallelism + shift_labels = shift_labels.to(shift_logits.device) + llm_loss = loss_fct(shift_logits, shift_labels) + + if is_eval: + loss = None + if not return_dict: + # print(f"!@#@#@#@#@#$#$#####$$$$$$$@#@##@##@#@#@###############@#@#@#@@@@@@@@@@@@@@{return_dict}@@@@@@@@@@@@") + output = (logits,) + outputs[1:] + return (loss,) + output if loss is not None else output + + return Qwen2VLCausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + rope_deltas=rope_deltas, + ) + + action_hidden_states = self.fusion_input_reasoning(labels=labels, input_ids=input_ids, + prev_layer_hidden_states=outputs.hidden_states, + hidden_states=hidden_states) + # else: + # action_hidden_states = hidden_states + external_obs_cond = None + if self.external_vision_encoder == 'resnet': + external_obs_cond = self.external_vision_encoder_model(raw_images) + + ret = self.policy_head(actions=actions, hidden_states=action_hidden_states, states=states, is_pad=is_pad, external_obs_cond=external_obs_cond) + + if self.with_llm_head: + loss = {'loss': ret['loss'] + self.llm_loss_weight * llm_loss, + 'llm_loss': llm_loss, + 'action_loss': ret['loss']} + else: + loss = {'loss': ret['loss'], + 'llm_loss': (torch.ones(1)*(-100)).to(ret['loss'].dtype).squeeze(0), + 'action_loss': ret['loss']} + if not return_dict: + output = (logits,) + outputs[1:] + return (loss,) + output if loss is not None else output + + torch.cuda.empty_cache() + gc.collect() + del input_ids + del attention_mask + del position_ids + del past_key_values + del inputs_embeds + del labels + del pixel_values + del image_grid_thw + del actions + del states + return Qwen2VLCausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + rope_deltas=rope_deltas, + ) + + def fusion_input_reasoning(self, labels, input_ids, hidden_states, prev_layer_hidden_states=None): + inputs_index = labels[:, :] == -100 + inputs_index = inputs_index.int() + # diff = inputs_index[:, :1] - inputs_index[:, 1:] + # indexs = torch.argmax((diff!=0).float(), dim=1) + xor_array = torch.bitwise_xor(inputs_index[:, :-1], inputs_index[:, 1:]) + indexs = torch.argmax((xor_array != 0).float(), dim=1) + input_embeddings = [] + reasoning_embeddings = [] + identity = [] + input_embeddings_first_layer = [] + for i in range(indexs.shape[0]): + end = indexs[i] + 1 + temp = input_ids[i] == 151643 # pad token id + start = sum(temp.int()) + input_embeddings.append(self.input_action_proj(hidden_states[i, start:end, :])) + identity.append(torch.mean(hidden_states[i, start:end, :], dim=0)) + + input_embeddings_first_layer.append(torch.mean(prev_layer_hidden_states[0][i, start:end, :], dim=0)) + + reasoning_embeddings.append(self.reasoning_action_proj(hidden_states[i, end:, :])) + input_embeddings = torch.cat(input_embeddings, dim=0) + input_embeddings_first_layer = torch.stack(input_embeddings_first_layer) + reasoning_embeddings = torch.cat(reasoning_embeddings, dim=0) + identity = torch.stack(identity) + + if self.using_film: + action_hidden_states = self.reasoning_film(input_embeddings, reasoning_embeddings).unsqueeze(1) + action_hidden_states = action_hidden_states + identity.unsqueeze(1) + else: + # action_hidden_states = input_embeddings.unsqueeze(1) + reasoning_embeddings.unsqueeze(1) + action_hidden_states = identity.unsqueeze(1) + + if self.using_first_layer_hidden_states: + action_hidden_states = action_hidden_states + input_embeddings_first_layer.unsqueeze(1) + return action_hidden_states + + def xattn_forward(self, labels, input_ids, hidden_states): + inputs_index = labels[:, :] == -100 + inputs_index = inputs_index.int() + # diff = inputs_index[:, :1] - inputs_index[:, 1:] + # indexs = torch.argmax((diff!=0).float(), dim=1) + xor_array = torch.bitwise_xor(inputs_index[:, :-1], inputs_index[:, 1:]) + indexs = torch.argmax((xor_array != 0).float(), dim=1) + input_embeddings = [] + reasoning_embeddings = [] + for i in range(indexs.shape[0]): + end = indexs[i] + 1 + temp = input_ids[i] == 151643 # pad token id + start = sum(temp.int()) + input_embeddings.append(torch.flip(hidden_states[i, start:end, :], dims=[1])) + reasoning_embeddings.append(torch.flip(hidden_states[i, end:, :], dims=[1])) + padd_value = 0 + + input_embeddings = torch.nn.utils.rnn.pad_sequence(input_embeddings, + batch_first=True, + padding_value=padd_value) + reasoning_embeddings = torch.nn.utils.rnn.pad_sequence(reasoning_embeddings, + batch_first=True, + padding_value=padd_value) + + input_embeddings = torch.flip(input_embeddings, dims=[1]) + reasoning_embeddings = torch.flip(reasoning_embeddings, dims=[1]) + + r_mask = reasoning_embeddings[..., 0].ne(padd_value).float() + i_mask = input_embeddings[..., 0].ne(padd_value).float() + x_attn_mask = torch.matmul(r_mask.unsqueeze(-1), i_mask.unsqueeze(-1).transpose(-1, -2)) + x_attn_mask = x_attn_mask.masked_fill(x_attn_mask == 0, float(-50)).masked_fill(x_attn_mask == 1, float(0.0)) + input_embeddings = input_embeddings.to(torch.bfloat16) + reasoning_embeddings = reasoning_embeddings.to(torch.bfloat16) + x_attn_mask = x_attn_mask.to(torch.bfloat16) + action_hidden_states = self.xattn(encoder_hidden_states=input_embeddings, hidden_states=reasoning_embeddings, + encoder_attention_mask=x_attn_mask)[0] + r_mask = r_mask.unsqueeze(-1).expand_as(action_hidden_states) + action_hidden_states = action_hidden_states * r_mask + sum_val = action_hidden_states.sum(dim=1) + counts = r_mask.sum(dim=1).float() + action_hidden_states = sum_val / (counts) + action_hidden_states = action_hidden_states.unsqueeze(1).to(torch.bfloat16) + # action_hidden_states = torch.mean(action_hidden_states, dim=1).unsqueeze(1) + return action_hidden_states + + def prepare_inputs_for_generation( + self, + input_ids, + past_key_values=None, + attention_mask=None, + inputs_embeds=None, + cache_position=None, + position_ids=None, + use_cache=True, + pixel_values=None, + pixel_values_videos=None, + image_grid_thw=None, + video_grid_thw=None, + **kwargs, + ): + # If we have cache: let's slice `input_ids` through `cache_position`, to keep only the unprocessed tokens + # Exception 1: when passing input_embeds, input_ids may be missing entries + # Exception 2: some generation methods do special slicing of input_ids, so we don't need to do it here + if past_key_values is not None: + if inputs_embeds is not None: # Exception 1 + input_ids = input_ids[:, -cache_position.shape[0]:] + elif input_ids.shape[1] != cache_position.shape[0]: # Default case (the "else", a no op, is Exception 2) + input_ids = input_ids[:, cache_position] + + rope_deltas = kwargs.get("rope_deltas", None) + if attention_mask is not None and position_ids is None: + if cache_position is None or (cache_position is not None and cache_position[0] == 0): + position_ids, rope_deltas = self.get_rope_index( + input_ids, image_grid_thw, video_grid_thw, attention_mask + ) + else: + batch_size, seq_length = input_ids.shape + delta = ( + cache_position[0] + rope_deltas if cache_position is not None and rope_deltas is not None else 0 + ) + position_ids = torch.arange(seq_length, device=input_ids.device) + position_ids = position_ids.view(1, -1).expand(batch_size, -1) + position_ids = position_ids.add(delta) + position_ids = position_ids.unsqueeze(0).expand(3, -1, -1) + + if cache_position[0] != 0: + pixel_values = None + pixel_values_videos = None + + # if `inputs_embeds` are passed, we only want to use them in the 1st generation step + if inputs_embeds is not None and cache_position[0] == 0: + model_inputs = {"inputs_embeds": inputs_embeds, "input_ids": None} + else: + model_inputs = {"input_ids": input_ids, "inputs_embeds": None} + + if isinstance(past_key_values, StaticCache) and attention_mask.ndim == 2: + if model_inputs["inputs_embeds"] is not None: + batch_size, sequence_length, _ = inputs_embeds.shape + device = inputs_embeds.device + else: + batch_size, sequence_length = input_ids.shape + device = input_ids.device + + attention_mask = self.model._prepare_4d_causal_attention_mask_with_cache_position( + attention_mask, + sequence_length=sequence_length, + target_length=past_key_values.get_max_cache_shape(), + dtype=self.lm_head.weight.dtype, + device=device, + cache_position=cache_position, + batch_size=batch_size, + config=self.config, + past_key_values=past_key_values, + ) + + model_inputs.update( + { + "position_ids": position_ids, + "past_key_values": past_key_values, + "use_cache": use_cache, + "attention_mask": attention_mask, + "pixel_values": pixel_values, + "pixel_values_videos": pixel_values_videos, + "image_grid_thw": image_grid_thw, + "video_grid_thw": video_grid_thw, + "rope_deltas": rope_deltas, + } + ) + model_inputs.update(kwargs) + return model_inputs + + def evaluate(self, + input_ids: torch.LongTensor = None, + actions=None, + states=None, + is_pad=None, + tokenizer=None, + is_eval=True, + select_one=False, + pixel_values=None, + policy_config=None, + attention_mask=None, + image_grid_thw=None, + raw_images=None, + ): + input_ids = input_ids.to('cuda') + if self.using_state: + state_id = torch.ones((input_ids.shape[0], 1), dtype=input_ids.dtype, device=input_ids.device) + input_ids = torch.cat([input_ids, state_id], dim=-1) + attn_mask_state_embedding = torch.ones((attention_mask.shape[0], 1), dtype=torch.bool, + device=attention_mask.device) + attention_mask = torch.cat([attention_mask, attn_mask_state_embedding], dim=-1) + + with torch.inference_mode(): + outputs = self.generate( + input_ids, + pixel_values=pixel_values, + attention_mask=attention_mask, + image_grid_thw=image_grid_thw, + is_eval=is_eval, + states=states.to(torch.bfloat16), + num_beams=1, + do_sample=False, + temperature=0.2, + max_new_tokens=60, + eos_token_id=tokenizer.eos_token_id, # End of sequence token + pad_token_id=tokenizer.eos_token_id, # Pad token + use_cache=True, + output_hidden_states=True, + return_dict_in_generate=True, + ) + + output_ids = outputs.sequences + # last_hidden_states = outputs.hidden_states[-2][-1] + input_token_len = input_ids.shape[1] + n_diff_input_output = (input_ids != output_ids[:, :input_token_len]).sum().item() + if n_diff_input_output > 0: + print(f'[Warning] {n_diff_input_output} output_ids are not the same as the input_ids') + outputs_text = tokenizer.batch_decode(output_ids[:, input_token_len:], skip_special_tokens=False)[0] + + outputs_text = outputs_text.strip() + new_output_ids = tokenizer.encode(outputs_text) + last_hidden_states = [each[-1] for each in outputs.hidden_states] # all hidden states + all_hidden_states = torch.cat(last_hidden_states, dim=1) + ground_truth_reasoning_embed = self.model.embed_tokens(torch.tensor(new_output_ids).to('cuda')).unsqueeze(0) + new_output_ids = torch.tensor([new_output_ids]).to(device=input_ids.device) + labels = torch.cat([torch.ones_like(input_ids) * -100, new_output_ids], dim=-1) + + action_hidden_states = None + external_obs_cond = None + if self.external_vision_encoder == 'resnet': + external_obs_cond = self.external_vision_encoder_model(raw_images) + action_hidden_states = self.fusion_input_reasoning(labels=labels, + input_ids=torch.cat([input_ids, new_output_ids], dim=-1), + hidden_states=torch.cat(last_hidden_states, dim=1), + prev_layer_hidden_states=outputs.hidden_states[0]) + + + # print(outputs) + action = self.policy_head(actions, action_hidden_states, states.to(all_hidden_states.dtype), is_pad, external_obs_cond=external_obs_cond) + return action, outputs_text + + def evaluate_tinyvla(self, + input_ids: torch.LongTensor = None, + actions=None, + states=None, + is_pad=None, + tokenizer=None, + is_eval=True, + select_one=False, + pixel_values=None, + policy_config=None, + attention_mask=None, + image_grid_thw=None, + ): + input_ids = input_ids.to('cuda') + with torch.inference_mode(): + all_hidden_states = self.forward(input_ids, + pixel_values=pixel_values, + attention_mask=attention_mask, + image_grid_thw=image_grid_thw, + is_eval=is_eval, + tinyvla=True) + + # last_hidden_states = outputs.hidden_states[-2][-1] + # select_one = False + all_hidden_states = torch.mean(all_hidden_states, dim=1).unsqueeze(1) + + # print(outputs) + action = self.policy_head(actions, all_hidden_states, states.to(all_hidden_states.dtype), is_pad) + return action, "tinyvla no output" + +from transformers import AutoModelForCausalLM +AutoModelForCausalLM.register(DexVLAConfig, DexVLA) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/models/qwen2_vl_modules.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/models/qwen2_vl_modules.py new file mode 100644 index 00000000..13576cdb --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/models/qwen2_vl_modules.py @@ -0,0 +1,1444 @@ +# coding=utf-8 +# Copyright 2024 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch Qwen2-VL model.""" + +import math +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint +from torch.nn import CrossEntropyLoss, LayerNorm + +from transformers.activations import ACT2FN +from transformers.cache_utils import Cache, SlidingWindowCache, StaticCache +from transformers.generation import GenerationMixin +from transformers.modeling_attn_mask_utils import ( + AttentionMaskConverter, +) +from transformers.modeling_outputs import ( + BaseModelOutputWithPast, + ModelOutput, +) + +from types import SimpleNamespace + +from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS +from transformers.modeling_utils import PreTrainedModel +from transformers.utils import ( + add_start_docstrings, + add_start_docstrings_to_model_forward, + is_flash_attn_2_available, + is_flash_attn_greater_or_equal_2_10, + logging, + replace_return_docstrings, +) +from .configuration_dex_vla import DexVLAConfig, Qwen2VLAVisionConfig +import gc + +if is_flash_attn_2_available(): + from flash_attn import flash_attn_varlen_func + + from transformers.modeling_flash_attention_utils import _flash_attention_forward +else: + flash_attn_varlen_func = None + +from transformers import AutoConfig, AutoModel + +logger = logging.get_logger(__name__) + +_CONFIG_FOR_DOC = "Qwen2VLConfig" + + +@dataclass +class Qwen2VLCausalLMOutputWithPast(ModelOutput): + """ + Base class for Qwen2VL causal language model (or autoregressive) outputs. + + Args: + loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): + Language modeling loss (for next-token prediction). + logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape + `(batch_size, num_heads, sequence_length, embed_size_per_head)`) + + Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see + `past_key_values` input) to speed up sequential decoding. + hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. + + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. + attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. + + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. + """ + + loss: Optional[torch.FloatTensor] = None + logits: torch.FloatTensor = None + past_key_values: Optional[List[torch.FloatTensor]] = None + hidden_states: Optional[Tuple[torch.FloatTensor]] = None + attentions: Optional[Tuple[torch.FloatTensor]] = None + rope_deltas: Optional[torch.LongTensor] = None + + +class Qwen2VLRotaryEmbedding(nn.Module): + def __init__( + self, + dim=None, + max_position_embeddings=2048, + base=10000, + device=None, + scaling_factor=1.0, + rope_type="default", + config: Optional[DexVLAConfig] = None, + ): + super().__init__() + # TODO (joao): remove the `if` below, only used for BC + self.rope_kwargs = {} + if config is None: + logger.warning_once( + "`Qwen2VLRotaryEmbedding` can now be fully parameterized by passing the model config through the " + "`config` argument. All other arguments will be removed in v4.46" + ) + self.rope_kwargs = { + "rope_type": rope_type, + "factor": scaling_factor, + "dim": dim, + "base": base, + "max_position_embeddings": max_position_embeddings, + } + self.rope_type = rope_type + self.max_seq_len_cached = max_position_embeddings + self.original_max_seq_len = max_position_embeddings + else: + # BC: "rope_type" was originally "type" + if config.rope_scaling is not None: + self.rope_type = config.rope_scaling.get("rope_type", config.rope_scaling.get("type")) + else: + self.rope_type = "default" + self.max_seq_len_cached = config.max_position_embeddings + self.original_max_seq_len = config.max_position_embeddings + + self.config = config + self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] + + inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device, **self.rope_kwargs) + self.register_buffer("inv_freq", inv_freq, persistent=False) + self.original_inv_freq = self.inv_freq + + def _dynamic_frequency_update(self, position_ids, device): + """ + dynamic RoPE layers should recompute `inv_freq` in the following situations: + 1 - growing beyond the cached sequence length (allow scaling) + 2 - the current sequence length is in the original scale (avoid losing precision with small sequences) + """ + seq_len = torch.max(position_ids) + 1 + if seq_len > self.max_seq_len_cached: # growth + inv_freq, self.attention_scaling = self.rope_init_fn( + self.config, device, seq_len=seq_len, **self.rope_kwargs + ) + self.register_buffer("inv_freq", inv_freq, persistent=False) # TODO joao: may break with compilation + self.max_seq_len_cached = seq_len + + if seq_len < self.original_max_seq_len and self.max_seq_len_cached > self.original_max_seq_len: # reset + self.register_buffer("inv_freq", self.original_inv_freq, persistent=False) + self.max_seq_len_cached = self.original_max_seq_len + + @torch.no_grad() + def forward(self, x, position_ids): + if "dynamic" in self.rope_type: + self._dynamic_frequency_update(position_ids, device=x.device) + + # Core RoPE block. In contrast to other models, Qwen2_VL has different position ids for thw grids + # So we expand the inv_freq to shape (3, ...) + inv_freq_expanded = self.inv_freq[None, None, :, None].float().expand(3, position_ids.shape[1], -1, 1) + position_ids_expanded = position_ids[:, :, None, :].float() # shape (3, bs, 1, positions) + # Force float32 (see https://github.com/huggingface/transformers/pull/29285) + device_type = x.device.type + device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu" + with torch.autocast(device_type=device_type, enabled=False): + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(2, 3) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() + sin = emb.sin() + + # Advanced RoPE types (e.g. yarn) apply a post-processing scaling factor, equivalent to scaling attention + cos = cos * self.attention_scaling + sin = sin * self.attention_scaling + + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +# Copied from transformers.models.llama.modeling_llama.rotate_half +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +def apply_multimodal_rotary_pos_emb(q, k, cos, sin, mrope_section, unsqueeze_dim=1): + """Applies Rotary Position Embedding with Multimodal Sections to the query and key tensors (https://qwenlm.github.io/blog/qwen2-vl/). + + Explanation: + Multimodal 3D rotary position embedding is an extension to 1D rotary position embedding. The input embedding + sequence contains vision (images / videos) embedding and text embedding or just contains text embedding. For + vision embedding part, we apply rotary position embedding on temporal, height and width dimension seperately. + Here we split the channel dimension to 3 chunks for the temporal, height and width rotary position embedding. + For text embedding part, we just apply 1D rotary position embedding. The three rotary position index (temporal, + height and width) of text embedding is always the same, so the text embedding rotary position embedding has no + difference with modern LLMs. + + Args: + q (`torch.Tensor`): The query tensor. + k (`torch.Tensor`): The key tensor. + cos (`torch.Tensor`): The cosine part of the rotary embedding. + sin (`torch.Tensor`): The sine part of the rotary embedding. + position_ids (`torch.Tensor`): + The position indices of the tokens corresponding to the query and key tensors. For example, this can be + used to pass offsetted position ids when working with a KV-cache. + mrope_section(`List(int)`): + Multimodal rope section is for channel dimension of temporal, height and width in rope calculation. + unsqueeze_dim (`int`, *optional*, defaults to 1): + The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and + sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note + that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and + k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes + cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have + the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. + Returns: + `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. + """ + mrope_section = mrope_section * 2 + cos = torch.cat([m[i % 3] for i, m in enumerate(cos.split(mrope_section, dim=-1))], dim=-1).unsqueeze( + unsqueeze_dim + ) + sin = torch.cat([m[i % 3] for i, m in enumerate(sin.split(mrope_section, dim=-1))], dim=-1).unsqueeze( + unsqueeze_dim + ) + + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + + +def apply_rotary_pos_emb_vision(tensor: torch.Tensor, freqs: torch.Tensor) -> torch.Tensor: + orig_dtype = tensor.dtype + tensor = tensor.float() + cos = freqs.cos() + sin = freqs.sin() + cos = cos.unsqueeze(1).repeat(1, 1, 2).unsqueeze(0).float() + sin = sin.unsqueeze(1).repeat(1, 1, 2).unsqueeze(0).float() + output = (tensor * cos) + (rotate_half(tensor) * sin) + output = output.to(orig_dtype) + return output + + +class VisionRotaryEmbedding(nn.Module): + def __init__(self, dim: int, theta: float = 10000.0) -> None: + super().__init__() + inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float) / dim)) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + def forward(self, seqlen: int) -> torch.Tensor: + seq = torch.arange(seqlen, device=self.inv_freq.device, dtype=self.inv_freq.dtype) + freqs = torch.outer(seq, self.inv_freq) + return freqs + + +class PatchEmbed(nn.Module): + def __init__( + self, + patch_size: int = 14, + temporal_patch_size: int = 2, + in_channels: int = 3, + embed_dim: int = 1152, + ) -> None: + super().__init__() + self.patch_size = patch_size + self.temporal_patch_size = temporal_patch_size + self.in_channels = in_channels + self.embed_dim = embed_dim + + kernel_size = [temporal_patch_size, patch_size, patch_size] + self.proj = nn.Conv3d(in_channels, embed_dim, kernel_size=kernel_size, stride=kernel_size, bias=False) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + target_dtype = self.proj.weight.dtype + hidden_states = hidden_states.view( + -1, self.in_channels, self.temporal_patch_size, self.patch_size, self.patch_size + ) + hidden_states = self.proj(hidden_states.to(dtype=target_dtype)).view(-1, self.embed_dim) + return hidden_states + + +class PatchMerger(nn.Module): + def __init__(self, dim: int, context_dim: int, spatial_merge_size: int = 2) -> None: + super().__init__() + self.hidden_size = context_dim * (spatial_merge_size**2) + self.ln_q = LayerNorm(context_dim, eps=1e-6) + self.mlp = nn.Sequential( + nn.Linear(self.hidden_size, self.hidden_size), + nn.GELU(), + nn.Linear(self.hidden_size, dim), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.mlp(self.ln_q(x).view(-1, self.hidden_size)) + return x + + +class VisionMlp(nn.Module): + def __init__(self, dim: int, hidden_dim: int, hidden_act: str) -> None: + super().__init__() + self.fc1 = nn.Linear(dim, hidden_dim) + self.act = ACT2FN[hidden_act] + self.fc2 = nn.Linear(hidden_dim, dim) + + def forward(self, x) -> torch.Tensor: + return self.fc2(self.act(self.fc1(x))) + + +class VisionAttention(nn.Module): + def __init__(self, dim: int, num_heads: int = 16) -> None: + super().__init__() + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.qkv = nn.Linear(dim, dim * 3, bias=True) + self.proj = nn.Linear(dim, dim) + + def forward( + self, hidden_states: torch.Tensor, cu_seqlens: torch.Tensor, rotary_pos_emb: torch.Tensor = None + ) -> torch.Tensor: + seq_length = hidden_states.shape[0] + q, k, v = self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0) + q = apply_rotary_pos_emb_vision(q.unsqueeze(0), rotary_pos_emb).squeeze(0) + k = apply_rotary_pos_emb_vision(k.unsqueeze(0), rotary_pos_emb).squeeze(0) + + attention_mask = torch.full( + [1, seq_length, seq_length], torch.finfo(q.dtype).min, device=q.device, dtype=q.dtype + ) + for i in range(1, len(cu_seqlens)): + attention_mask[..., cu_seqlens[i - 1] : cu_seqlens[i], cu_seqlens[i - 1] : cu_seqlens[i]] = 0 + + q = q.transpose(0, 1) + k = k.transpose(0, 1) + v = v.transpose(0, 1) + attn_weights = torch.matmul(q, k.transpose(1, 2)) / math.sqrt(self.head_dim) + attn_weights = attn_weights + attention_mask + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(q.dtype) + attn_output = torch.matmul(attn_weights, v) + attn_output = attn_output.transpose(0, 1) + attn_output = attn_output.reshape(seq_length, -1) + attn_output = self.proj(attn_output) + return attn_output + + +class VisionFlashAttention2(nn.Module): + def __init__(self, dim: int, num_heads: int = 16) -> None: + super().__init__() + self.num_heads = num_heads + self.qkv = nn.Linear(dim, dim * 3, bias=True) + self.proj = nn.Linear(dim, dim) + + def forward( + self, hidden_states: torch.Tensor, cu_seqlens: torch.Tensor, rotary_pos_emb: torch.Tensor = None + ) -> torch.Tensor: + seq_length = hidden_states.shape[0] + q, k, v = self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0) + q = apply_rotary_pos_emb_vision(q.unsqueeze(0), rotary_pos_emb).squeeze(0) + k = apply_rotary_pos_emb_vision(k.unsqueeze(0), rotary_pos_emb).squeeze(0) + + max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item() + attn_output = flash_attn_varlen_func(q, k, v, cu_seqlens, cu_seqlens, max_seqlen, max_seqlen).reshape( + seq_length, -1 + ) + attn_output = self.proj(attn_output) + return attn_output + + +class VisionSdpaAttention(nn.Module): + def __init__(self, dim: int, num_heads: int = 16) -> None: + super().__init__() + self.num_heads = num_heads + self.qkv = nn.Linear(dim, dim * 3, bias=True) + self.proj = nn.Linear(dim, dim) + + def forward( + self, hidden_states: torch.Tensor, cu_seqlens: torch.Tensor, rotary_pos_emb: torch.Tensor = None + ) -> torch.Tensor: + seq_length = hidden_states.shape[0] + q, k, v = self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0) + q = apply_rotary_pos_emb_vision(q.unsqueeze(0), rotary_pos_emb).squeeze(0) + k = apply_rotary_pos_emb_vision(k.unsqueeze(0), rotary_pos_emb).squeeze(0) + + attention_mask = torch.zeros([1, seq_length, seq_length], device=q.device, dtype=torch.bool) + for i in range(1, len(cu_seqlens)): + attention_mask[..., cu_seqlens[i - 1] : cu_seqlens[i], cu_seqlens[i - 1] : cu_seqlens[i]] = True + q = q.transpose(0, 1) + k = k.transpose(0, 1) + v = v.transpose(0, 1) + attn_output = F.scaled_dot_product_attention(q, k, v, attention_mask, dropout_p=0.0) + attn_output = attn_output.transpose(0, 1) + attn_output = attn_output.reshape(seq_length, -1) + attn_output = self.proj(attn_output) + return attn_output + + +QWEN2_VL_VISION_ATTENTION_CLASSES = { + "eager": VisionAttention, + "flash_attention_2": VisionFlashAttention2, + "sdpa": VisionSdpaAttention, +} + + +class Qwen2VLVisionBlock(nn.Module): + def __init__(self, config, attn_implementation: str = "sdpa") -> None: + super().__init__() + self.norm1 = LayerNorm(config.embed_dim, eps=1e-6) + self.norm2 = LayerNorm(config.embed_dim, eps=1e-6) + mlp_hidden_dim = int(config.embed_dim * config.mlp_ratio) + + self.attn = QWEN2_VL_VISION_ATTENTION_CLASSES[attn_implementation]( + config.embed_dim, num_heads=config.num_heads + ) + self.mlp = VisionMlp(dim=config.embed_dim, hidden_dim=mlp_hidden_dim, hidden_act=config.hidden_act) + + def forward(self, hidden_states, cu_seqlens, rotary_pos_emb) -> torch.Tensor: + hidden_states = hidden_states + self.attn( + self.norm1(hidden_states).to(torch.bfloat16), cu_seqlens=cu_seqlens, rotary_pos_emb=rotary_pos_emb + ) + hidden_states = hidden_states + self.mlp(self.norm2(hidden_states)) + return hidden_states + + +# Copied from transformers.models.qwen2.modeling_qwen2.Qwen2RMSNorm +class Qwen2RMSNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + """ + Qwen2RMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + def extra_repr(self): + return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" + + +# Copied from transformers.models.qwen2.modeling_qwen2.Qwen2MLP +class Qwen2MLP(nn.Module): + def __init__(self, config): + super().__init__() + self.hidden_size = config.hidden_size + self.intermediate_size = config.intermediate_size + self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) + self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) + self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False) + self.act_fn = ACT2FN[config.hidden_act] + + def forward(self, hidden_state): + return self.down_proj(self.act_fn(self.gate_proj(hidden_state)) * self.up_proj(hidden_state)) + + +# Copied from transformers.models.llama.modeling_llama.repeat_kv +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + +class Qwen2VLAttention(nn.Module): + """ + Multi-headed attention from 'Attention Is All You Need' paper. Modified to use sliding window attention: Longformer + and "Generating Long Sequences with Sparse Transformers". + """ + + def __init__(self, config: DexVLAConfig, layer_idx: Optional[int] = None): + super().__init__() + self.config = config + self.layer_idx = layer_idx + if layer_idx is None: + logger.warning_once( + f"Instantiating {self.__class__.__name__} without passing `layer_idx` is not recommended and will " + "to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` " + "when creating this class." + ) + + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.num_key_value_heads = config.num_key_value_heads + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.max_position_embeddings = config.max_position_embeddings + self.rope_theta = config.rope_theta + self.is_causal = True + self.attention_dropout = config.attention_dropout + self.rope_scaling = config.rope_scaling + + if (self.head_dim * self.num_heads) != self.hidden_size: + raise ValueError( + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" + f" and `num_heads`: {self.num_heads})." + ) + self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=True) + self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True) + self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True) + self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) + + self.rotary_emb = Qwen2VLRotaryEmbedding( + self.head_dim, + max_position_embeddings=self.max_position_embeddings, + base=self.rope_theta, + ) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + cache_position: Optional[torch.LongTensor] = None, + position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, # will become mandatory in v4.46 + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + kv_seq_len += cache_position[0] + 1 + + if position_embeddings is None: + logger.warning_once( + "The attention layers in this model are transitioning from computing the RoPE embeddings internally " + "through `position_ids` (2D tensor with the indexes of the tokens), to using externally computed " + "`position_embeddings` (Tuple of tensors, containing cos and sin). In v4.46 `position_ids` will be " + "removed and `position_embeddings` will be mandatory." + ) + cos, sin = self.rotary_emb(value_states, position_ids) + else: + cos, sin = position_embeddings + query_states, key_states = apply_multimodal_rotary_pos_emb( + query_states, key_states, cos, sin, self.rope_scaling["mrope_section"] + ) + + if past_key_value is not None: + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} # Specific to RoPE models + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + # repeat k/v heads if n_kv_heads < n_heads + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + + if attention_mask is not None: # no matter the length, we just slice it + causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] + attn_weights = attn_weights + causal_mask + + # Fix precision issues in Qwen2-VL float16 inference + # Replace inf values with zeros in attention weights to prevent NaN propagation + if query_states.dtype == torch.float16: + attn_weights = torch.where(torch.isinf(attn_weights), torch.zeros_like(attn_weights), attn_weights) + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) + attn_output = torch.matmul(attn_weights, value_states) + + if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.reshape(bsz, q_len, -1) + + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + +class Qwen2VLFlashAttention2(Qwen2VLAttention): + """ + Qwen2VL flash attention module, following Qwen2VL attention module. This module inherits from `Qwen2VLAttention` + as the weights of the module stays untouched. The only required change would be on the forward pass + where it needs to correctly call the public API of flash attention and deal with padding tokens + in case the input contains any of them. Additionally, for sliding window attention, we apply SWA only to the bottom + config.max_window_layers layers. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. + # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. + # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). + self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + cache_position: Optional[torch.LongTensor] = None, + position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, # will become mandatory in v4.46 + ): + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + if self.layer_idx is None: + raise ValueError( + f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} " + "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class " + "with a layer index." + ) + kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) + + # Because the input can be padded, the absolute sequence length depends on the max position id. + if position_embeddings is None: + logger.warning_once( + "The attention layers in this model are transitioning from computing the RoPE embeddings internally " + "through `position_ids` (2D tensor with the indexes of the tokens), to using externally computed " + "`position_embeddings` (Tuple of tensors, containing cos and sin). In v4.46 `position_ids` will be " + "removed and `position_embeddings` will be mandatory." + ) + cos, sin = self.rotary_emb(value_states, position_ids) + else: + cos, sin = position_embeddings + + query_states, key_states = apply_multimodal_rotary_pos_emb( + query_states, key_states, cos, sin, self.rope_scaling["mrope_section"] + ) + + if past_key_value is not None: + # Activate slicing cache only if the config has a value `sliding_windows` attribute + cache_has_contents = past_key_value.get_seq_length(self.layer_idx) > 0 + if ( + getattr(self.config, "sliding_window", None) is not None + and kv_seq_len > self.config.sliding_window + and cache_has_contents + ): + slicing_tokens = 1 - self.config.sliding_window + + past_key = past_key_value[self.layer_idx][0] + past_value = past_key_value[self.layer_idx][1] + + past_key = past_key[:, :, slicing_tokens:, :].contiguous() + past_value = past_value[:, :, slicing_tokens:, :].contiguous() + + if past_key.shape[-2] != self.config.sliding_window - 1: + raise ValueError( + f"past key must have a shape of (`batch_size, num_heads, self.config.sliding_window-1, head_dim`), got" + f" {past_key.shape}" + ) + + if attention_mask is not None: + attention_mask = attention_mask[:, slicing_tokens:] + attention_mask = torch.cat([attention_mask, torch.ones_like(attention_mask[:, -1:])], dim=-1) + + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} # Specific to RoPE models + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + # repeat k/v heads if n_kv_heads < n_heads + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + dropout_rate = 0.0 if not self.training else self.attention_dropout + + # In PEFT, usually we cast the layer norms in float32 for training stability reasons + # therefore the input hidden states gets silently casted in float32. Hence, we need + # cast them back in float16 just to be sure everything works as expected. + input_dtype = query_states.dtype + if input_dtype == torch.float32: + if torch.is_autocast_enabled(): + target_dtype = torch.get_autocast_gpu_dtype() + # Handle the case where the model is quantized + elif hasattr(self.config, "_pre_quantization_dtype"): + target_dtype = self.config._pre_quantization_dtype + else: + target_dtype = self.q_proj.weight.dtype + + logger.warning_once( + f"The input hidden states seems to be silently casted in float32, this might be related to" + f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in" + f" {target_dtype}." + ) + + query_states = query_states.to(target_dtype) + key_states = key_states.to(target_dtype) + value_states = value_states.to(target_dtype) + + # Reashape to the expected shape for Flash Attention + query_states = query_states.transpose(1, 2) + key_states = key_states.transpose(1, 2) + value_states = value_states.transpose(1, 2) + + if ( + self.config.use_sliding_window + and getattr(self.config, "sliding_window", None) is not None + and self.layer_idx >= self.config.max_window_layers + ): + sliding_window = self.config.sliding_window + else: + sliding_window = None + + attn_output = _flash_attention_forward( + query_states, + key_states, + value_states, + attention_mask, + q_len, + dropout=dropout_rate, + sliding_window=sliding_window, + is_causal=self.is_causal, + use_top_left_mask=self._flash_attn_uses_top_left_mask, + ) + + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous() + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + +class Qwen2VLSdpaAttention(Qwen2VLAttention): + """ + Qwen2 attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from + `Qwen2Attention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to + SDPA API. + """ + + # Adapted from Qwen2Attention.forward + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + cache_position: Optional[torch.LongTensor] = None, + position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, # will become mandatory in v4.46 + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + if output_attentions: + # TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented. + logger.warning_once( + "Qwen2VLModel is using Qwen2VLSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, " + 'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' + ) + return super().forward( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + cache_position=cache_position, + ) + + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx) + if position_embeddings is None: + logger.warning_once( + "The attention layers in this model are transitioning from computing the RoPE embeddings internally " + "through `position_ids` (2D tensor with the indexes of the tokens), to using externally computed " + "`position_embeddings` (Tuple of tensors, containing cos and sin). In v4.46 `position_ids` will be " + "removed and `position_embeddings` will be mandatory." + ) + cos, sin = self.rotary_emb(value_states, position_ids) + else: + cos, sin = position_embeddings + query_states, key_states = apply_multimodal_rotary_pos_emb( + query_states, key_states, cos, sin, self.rope_scaling["mrope_section"] + ) + + if past_key_value is not None: + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} # Specific to RoPE models + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + causal_mask = attention_mask + if attention_mask is not None: # no matter the length, we just slice it + causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] + + # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask, + # Reference: https://github.com/pytorch/pytorch/issues/112577. + if query_states.device.type == "cuda" and attention_mask is not None: + query_states = query_states.contiguous() + key_states = key_states.contiguous() + value_states = value_states.contiguous() + + # We dispatch to SDPA's Flash Attention or Efficient kernels via this `is_causal` if statement instead of an inline conditional assignment + # in SDPA to support both torch.compile's dynamic shapes and full graph options. An inline conditional prevents dynamic shapes from compiling. + # The q_len > 1 is necessary to match with AttentionMaskConverter.to_causal_4d that does not create a causal mask in case q_len == 1. + is_causal = True if causal_mask is None and q_len > 1 else False + + attn_output = torch.nn.functional.scaled_dot_product_attention( + query_states, + key_states, + value_states, + attn_mask=causal_mask, + dropout_p=self.attention_dropout if self.training else 0.0, + is_causal=is_causal, + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.view(bsz, q_len, self.hidden_size) + + attn_output = self.o_proj(attn_output) + + return attn_output, None, past_key_value + + +QWEN2_VL_ATTENTION_CLASSES = { + "eager": Qwen2VLAttention, + "flash_attention_2": Qwen2VLFlashAttention2, + "sdpa": Qwen2VLSdpaAttention, +} + + +class Qwen2VLDecoderLayer(nn.Module): + def __init__(self, config: DexVLAConfig, layer_idx: int): + super().__init__() + self.hidden_size = config.hidden_size + + if config.use_sliding_window and config._attn_implementation != "flash_attention_2": + logger.warning_once( + f"Sliding Window Attention is enabled but not implemented for `{config._attn_implementation}`; " + "unexpected results may be encountered." + ) + self.self_attn = QWEN2_VL_ATTENTION_CLASSES[config._attn_implementation](config, layer_idx) + + self.mlp = Qwen2MLP(config) + self.input_layernorm = Qwen2RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = Qwen2RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: Optional[bool] = False, + use_cache: Optional[bool] = False, + cache_position: Optional[torch.LongTensor] = None, + position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, # will become mandatory in v4.46 + **kwargs, + ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`, *optional*): attention mask of size + `(batch, sequence_length)` where padding elements are indicated by 0. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states + cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): + Indices depicting the position of the input sequence tokens in the sequence. + position_embeddings (`Tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*): + Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`, + with `head_dim` being the embedding dimension of each attention head. + kwargs (`dict`, *optional*): + Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code + into the model + """ + + residual = hidden_states + + hidden_states = self.input_layernorm(hidden_states) + + # Self Attention + hidden_states, self_attn_weights, present_key_value = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + cache_position=cache_position, + position_embeddings=position_embeddings, + ) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + outputs = (hidden_states,) + + if output_attentions: + outputs += (self_attn_weights,) + + if use_cache: + outputs += (present_key_value,) + + return outputs + + +QWEN2VL_START_DOCSTRING = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`Qwen2VLConfig`]): + Model configuration class with all the parameters of the model. Initializing with a config file does not + load the weights associated with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + + +@add_start_docstrings( + "The bare Qwen2VL Model outputting raw hidden-states without any specific head on top.", + QWEN2VL_START_DOCSTRING, +) +class Qwen2VLPreTrainedModel(PreTrainedModel): + config_class = DexVLAConfig + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["Qwen2VLDecoderLayer", "Qwen2VLVisionBlock", "policy_head"] + _skip_keys_device_placement = "past_key_values" + _supports_flash_attn_2 = True + _supports_sdpa = True + _supports_cache_class = True + _supports_static_cache = True + + def _init_weights(self, module): + std = self.config.initializer_range + if isinstance(module, (nn.Linear, nn.Conv3d)): + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=std) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + +class Qwen2VisionTransformerPretrainedModel(Qwen2VLPreTrainedModel): + config_class = Qwen2VLAVisionConfig + _no_split_modules = ["Qwen2VLVisionBlock"] + + def __init__(self, config) -> None: + super().__init__(config) + self.spatial_merge_size = config.spatial_merge_size + + self.patch_embed = PatchEmbed( + patch_size=config.patch_size, + temporal_patch_size=config.temporal_patch_size, + in_channels=config.in_channels, + embed_dim=config.embed_dim, + ) + + head_dim = config.embed_dim // config.num_heads + self.rotary_pos_emb = VisionRotaryEmbedding(head_dim // 2) + + self.blocks = nn.ModuleList( + [Qwen2VLVisionBlock(config, config._attn_implementation) for _ in range(config.depth)] + ) + self.merger = PatchMerger( + dim=config.hidden_size, context_dim=config.embed_dim, spatial_merge_size=config.spatial_merge_size + ) + + def get_dtype(self) -> torch.dtype: + return self.blocks[0].mlp.fc2.weight.dtype + + def get_device(self) -> torch.device: + return self.blocks[0].mlp.fc2.weight.device + + def rot_pos_emb(self, grid_thw): + pos_ids = [] + for t, h, w in grid_thw: + hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w) + hpos_ids = hpos_ids.reshape( + h // self.spatial_merge_size, + self.spatial_merge_size, + w // self.spatial_merge_size, + self.spatial_merge_size, + ) + hpos_ids = hpos_ids.permute(0, 2, 1, 3) + hpos_ids = hpos_ids.flatten() + + wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1) + wpos_ids = wpos_ids.reshape( + h // self.spatial_merge_size, + self.spatial_merge_size, + w // self.spatial_merge_size, + self.spatial_merge_size, + ) + wpos_ids = wpos_ids.permute(0, 2, 1, 3) + wpos_ids = wpos_ids.flatten() + pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1)) + pos_ids = torch.cat(pos_ids, dim=0) + max_grid_size = grid_thw[:, 1:].max() + rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size) + rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) + return rotary_pos_emb + + def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor) -> torch.Tensor: + hidden_states = self.patch_embed(hidden_states) + rotary_pos_emb = self.rot_pos_emb(grid_thw) + + cu_seqlens = torch.repeat_interleave(grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]).cumsum( + dim=0, dtype=torch.int32 + ) + cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0) + + for blk in self.blocks: + hidden_states = blk(hidden_states, cu_seqlens=cu_seqlens, rotary_pos_emb=rotary_pos_emb) + + return self.merger(hidden_states) + + +@add_start_docstrings( + "The bare Qwen2VL Model outputting raw hidden-states without any specific head on top.", + QWEN2VL_START_DOCSTRING, +) +class Qwen2VLModel(Qwen2VLPreTrainedModel): + def __init__(self, config: DexVLAConfig): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + self.layers = nn.ModuleList( + [Qwen2VLDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] + ) + self._attn_implementation = config._attn_implementation + self.norm = Qwen2RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.rotary_emb = Qwen2VLRotaryEmbedding(config=config) + + self.gradient_checkpointing = False + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.embed_tokens + + def set_input_embeddings(self, value): + self.embed_tokens = value + + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + ) -> Union[Tuple, BaseModelOutputWithPast]: + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError("You must specify exactly one of input_ids or inputs_embeds") + + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + ) + use_cache = False + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + + if cache_position is None: + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + cache_position = torch.arange( + past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device + ) + + # the hard coded `3` is for temporal, height and width. + if position_ids is None: + position_ids = cache_position.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) + elif position_ids.dim() == 2: + position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) + + causal_mask = self._update_causal_mask( + attention_mask, inputs_embeds, cache_position, past_key_values, output_attentions + ) + + hidden_states = inputs_embeds + + # create position embeddings to be shared across the decoder layers + position_embeddings = self.rotary_emb(hidden_states, position_ids) + + # decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + next_decoder_cache = None + + for decoder_layer in self.layers: + if output_hidden_states: + all_hidden_states += (hidden_states,) + + if self.gradient_checkpointing and self.training: + layer_outputs = self._gradient_checkpointing_func( + decoder_layer.__call__, + hidden_states, + causal_mask, + position_ids, + past_key_values, + output_attentions, + use_cache, + cache_position, + position_embeddings, + ) + else: + layer_outputs = decoder_layer( + hidden_states, + attention_mask=causal_mask, + position_ids=position_ids, + past_key_value=past_key_values, + output_attentions=output_attentions, + use_cache=use_cache, + cache_position=cache_position, + position_embeddings=position_embeddings, + ) + + hidden_states = layer_outputs[0] + + if use_cache: + next_decoder_cache = layer_outputs[2 if output_attentions else 1] + + if output_attentions: + all_self_attns += (layer_outputs[1],) + + hidden_states = self.norm(hidden_states) + + # add hidden states from the last decoder layer + if output_hidden_states: + all_hidden_states += (hidden_states,) + + next_cache = next_decoder_cache if use_cache else None + + if not return_dict: + return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None) + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=next_cache, + hidden_states=all_hidden_states, + attentions=all_self_attns, + ) + + # Copied from transformers.models.phi3.modeling_phi3.Phi3Model._update_causal_mask + def _update_causal_mask( + self, + attention_mask: torch.Tensor, + input_tensor: torch.Tensor, + cache_position: torch.Tensor, + past_key_values: Cache, + output_attentions: bool, + ): + if self.config._attn_implementation == "flash_attention_2": + if attention_mask is not None and 0.0 in attention_mask: + return attention_mask + return None + + # For SDPA, when possible, we will rely on its `is_causal` argument instead of its `attn_mask` argument, in + # order to dispatch on Flash Attention 2. This feature is not compatible with static cache, as SDPA will fail + # to infer the attention mask. + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + using_static_cache = isinstance(past_key_values, StaticCache) + using_sliding_window_cache = isinstance(past_key_values, SlidingWindowCache) + + # When output attentions is True, sdpa implementation's forward method calls the eager implementation's forward + if ( + self.config._attn_implementation == "sdpa" + and not (using_static_cache or using_sliding_window_cache) + and not output_attentions + ): + if AttentionMaskConverter._ignore_causal_mask_sdpa( + attention_mask, + inputs_embeds=input_tensor, + past_key_values_length=past_seen_tokens, + sliding_window=self.config.sliding_window, + is_training=self.training, + ): + return None + + dtype, device = input_tensor.dtype, input_tensor.device + min_dtype = torch.finfo(dtype).min + sequence_length = input_tensor.shape[1] + # SlidingWindowCache or StaticCache + if using_sliding_window_cache or using_static_cache: + target_length = past_key_values.get_max_cache_shape() + # DynamicCache or no cache + else: + target_length = ( + attention_mask.shape[-1] + if isinstance(attention_mask, torch.Tensor) + else past_seen_tokens + sequence_length + 1 + ) + + # In case the provided `attention` mask is 2D, we generate a causal mask here (4D). + causal_mask = self._prepare_4d_causal_attention_mask_with_cache_position( + attention_mask, + sequence_length=sequence_length, + target_length=target_length, + dtype=dtype, + device=device, + cache_position=cache_position, + batch_size=input_tensor.shape[0], + config=self.config, + past_key_values=past_key_values, + ) + + if ( + self.config._attn_implementation == "sdpa" + and attention_mask is not None + and attention_mask.device.type == "cuda" + and not output_attentions + ): + # Attend to all tokens in fully masked rows in the causal_mask, for example the relevant first rows when + # using left padding. This is required by F.scaled_dot_product_attention memory-efficient attention path. + # Details: https://github.com/pytorch/pytorch/issues/110213 + causal_mask = AttentionMaskConverter._unmask_unattended(causal_mask, min_dtype) + + return causal_mask + + @staticmethod + # Copied from transformers.models.mistral.modeling_mistral.MistralModel._prepare_4d_causal_attention_mask_with_cache_position with Mistral->Qwen2VL + def _prepare_4d_causal_attention_mask_with_cache_position( + attention_mask: torch.Tensor, + sequence_length: int, + target_length: int, + dtype: torch.dtype, + device: torch.device, + cache_position: torch.Tensor, + batch_size: int, + config: DexVLAConfig, + past_key_values: Cache, + ): + """ + Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape + `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing. + + Args: + attention_mask (`torch.Tensor`): + A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape `(batch_size, 1, query_length, key_value_length)`. + sequence_length (`int`): + The sequence length being processed. + target_length (`int`): + The target length: when generating with static cache, the mask should be as long as the static cache, to account for the 0 padding, the part of the cache that is not filled yet. + dtype (`torch.dtype`): + The dtype to use for the 4D attention mask. + device (`torch.device`): + The device to plcae the 4D attention mask on. + cache_position (`torch.Tensor`): + Indices depicting the position of the input sequence tokens in the sequence. + batch_size (`torch.Tensor`): + Batch size. + config (`Qwen2VLConfig`): + The model's configuration class + past_key_values (`Cache`): + The cache class that is being used currently to generate + """ + #print('@'*50) + #print(attention_mask.shape) + if attention_mask is not None and attention_mask.dim() == 4: + # In this case we assume that the mask comes already in inverted form and requires no inversion or slicing. + causal_mask = attention_mask + else: + min_dtype = torch.finfo(dtype).min + causal_mask = torch.full( + (sequence_length, target_length), fill_value=min_dtype, dtype=dtype, device=device + ) + diagonal_attend_mask = torch.arange(target_length, device=device) > cache_position.reshape(-1, 1) + if config.sliding_window is not None: + # if we have sliding window, we should not attend to tokens beyond sliding window length, so we mask them out also + # the check is needed to verify is current checkpoint was trained with sliding window or not + if not isinstance(past_key_values, SlidingWindowCache) or sequence_length > target_length: + sliding_attend_mask = torch.arange(target_length, device=device) <= ( + cache_position.reshape(-1, 1) - config.sliding_window + ) + diagonal_attend_mask |= sliding_attend_mask + causal_mask *= diagonal_attend_mask + causal_mask = causal_mask[None, None, :, :].expand(batch_size, 1, -1, -1) + if attention_mask is not None: + causal_mask = causal_mask.clone() # copy to contiguous memory for in-place edit + if attention_mask.shape[-1] > target_length: + attention_mask = attention_mask[:, :target_length] + mask_length = attention_mask.shape[-1] + padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :] + padding_mask = padding_mask == 0 + causal_mask[:, :, :, :mask_length] = causal_mask[:, :, :, :mask_length].masked_fill( + padding_mask, min_dtype + ) + return causal_mask + + +QWEN2_VL_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape + `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape + `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. + + Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention + blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that + don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all + `decoder_input_ids` of shape `(batch_size, sequence_length)`. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + pixel_values (`torch.FloatTensor` of shape `(seq_length, num_channels * image_size * image_size)): + The tensors corresponding to the input images. Pixel values can be obtained using + [`AutoImageProcessor`]. See [`Qwen2VLImageProcessor.__call__`] for details. [`Qwen2VLProcessor`] uses + [`Qwen2VLImageProcessor`] for processing images. + pixel_values_videos (`torch.FloatTensor` of shape `(seq_length, num_channels * temporal_size * image_size * image_size)): + The tensors corresponding to the input videos. Pixel values can be obtained using + [`AutoImageProcessor`]. See [`Qwen2VLImageProcessor.__call__`] for details. [`Qwen2VLProcessor`] uses + [`Qwen2VLImageProcessor`] for processing videos. + image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): + The temporal, height and width of feature shape of each image in LLM. + video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): + The temporal, height and width of feature shape of each video in LLM. + rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*): + The rope index difference between sequence length and multimodal rope. +""" \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/train/dex_vla_trainer.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/train/dex_vla_trainer.py new file mode 100644 index 00000000..4d1eea58 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/train/dex_vla_trainer.py @@ -0,0 +1,1272 @@ +import os +import torch +import torch.nn as nn + +from torch.utils.data import Sampler, DataLoader, BatchSampler, Dataset + +from transformers.trainer import * +from diffusers.training_utils import EMAModel +import math +import sys +from transformers import Trainer +from transformers.trainer import ( + is_sagemaker_mp_enabled, + get_parameter_names, + has_length, + ALL_LAYERNORM_LAYERS, + logger, +) +from typing import List, Optional, Dict +from transformers.utils import is_torch_tpu_available +from transformers.trainer_pt_utils import get_dataloader_sampler + +def maybe_zero_3(param, ignore_status=False, name=None): + from deepspeed import zero + from deepspeed.runtime.zero.partition_parameters import ZeroParamStatus + if hasattr(param, "ds_id"): + if param.ds_status == ZeroParamStatus.NOT_AVAILABLE: + if not ignore_status: + print(name, 'no ignore status') + with zero.GatheredParameters([param]): + param = param.data.detach().cpu().clone() + else: + param = param.detach().cpu().clone() + return param + + +def get_mm_adapter_state_maybe_zero_3(named_params, keys_to_match): + to_return = {k: t for k, t in named_params if any(key_match in k for key_match in keys_to_match)} + to_return = {k: maybe_zero_3(v, ignore_status=True, name=k).cpu() for k, v in to_return.items()} + return to_return + + +def split_to_even_chunks(indices, lengths, num_chunks): + """ + Split a list of indices into `chunks` chunks of roughly equal lengths. + """ + + if len(indices) % num_chunks != 0: + return [indices[i::num_chunks] for i in range(num_chunks)] + + num_indices_per_chunk = len(indices) // num_chunks + + chunks = [[] for _ in range(num_chunks)] + chunks_lengths = [0 for _ in range(num_chunks)] + for index in indices: + shortest_chunk = chunks_lengths.index(min(chunks_lengths)) + chunks[shortest_chunk].append(index) + chunks_lengths[shortest_chunk] += lengths[index] + if len(chunks[shortest_chunk]) == num_indices_per_chunk: + chunks_lengths[shortest_chunk] = float("inf") + + return chunks + + +def get_modality_length_grouped_indices(lengths, batch_size, world_size, generator=None): + # We need to use torch for the random part as a distributed sampler will set the random seed for torch. + assert all(l != 0 for l in lengths), "Should not have zero length." + # assert all(l > 0 for l in lengths) or all(l < 0 for l in lengths), "Should have only positive or negative lengths." + + mm_indices, mm_lengths = zip(*[(i, l) for i, l in enumerate(lengths) if l > 0]) + # print(len(lengths),lengths) + # exit(0) + lang_indices, lang_lengths = zip(*[(i, -l) for i, l in enumerate(lengths) if l < 0]) + + assert len(mm_indices) > 0, "Should have at least one multimodal sample." + assert len(lang_indices) > 0, "Should have at least one language sample." + + mm_shuffle = [mm_indices[i] for i in get_length_grouped_indices(mm_lengths, batch_size, world_size, generator=None)] + lang_shuffle = [lang_indices[i] for i in + get_length_grouped_indices(lang_lengths, batch_size, world_size, generator=None)] + megabatch_size = world_size * batch_size + mm_megabatches = [mm_shuffle[i: i + megabatch_size] for i in range(0, len(mm_shuffle), megabatch_size)] + lang_megabatches = [lang_shuffle[i: i + megabatch_size] for i in range(0, len(lang_shuffle), megabatch_size)] + + last_mm = mm_megabatches[-1] + last_lang = lang_megabatches[-1] + additional_batch = last_mm + last_lang + megabatches = mm_megabatches[:-1] + lang_megabatches[:-1] + megabatch_indices = torch.randperm(len(megabatches), generator=generator) + megabatches = [megabatches[i] for i in megabatch_indices] + + if len(additional_batch) >= megabatch_size: + megabatches = [additional_batch[:megabatch_size]] + megabatches + additional_batch = additional_batch[megabatch_size:] + + if len(additional_batch) > 0: + megabatches.append(additional_batch) + + return [i for megabatch in megabatches for i in megabatch] + + +def get_length_grouped_indices(lengths, batch_size, world_size, generator=None, merge=True): + # We need to use torch for the random part as a distributed sampler will set the random seed for torch. + indices = torch.randperm(len(lengths), generator=generator) + megabatch_size = world_size * batch_size + megabatches = [indices[i: i + megabatch_size].tolist() for i in range(0, len(lengths), megabatch_size)] + megabatches = [sorted(megabatch, key=lambda i: lengths[i], reverse=True) for megabatch in megabatches] + megabatches = [split_to_even_chunks(megabatch, lengths, world_size) for megabatch in megabatches] + + return [i for megabatch in megabatches for batch in megabatch for i in batch] + + +class LengthGroupedSampler(Sampler): + r""" + Sampler that samples indices in a way that groups together features of the dataset of roughly the same length while + keeping a bit of randomness. + """ + + def __init__( + self, + batch_size: int, + world_size: int, + lengths: Optional[List[int]] = None, + generator=None, + group_by_modality: bool = False, + ): + if lengths is None: + raise ValueError("Lengths must be provided.") + + self.batch_size = batch_size + self.world_size = world_size + self.lengths = lengths + self.generator = generator + self.group_by_modality = group_by_modality + + def __len__(self): + return len(self.lengths) + + def __iter__(self): + if self.group_by_modality: + indices = get_modality_length_grouped_indices(self.lengths, self.batch_size, self.world_size, + generator=self.generator) + else: + indices = get_length_grouped_indices(self.lengths, self.batch_size, self.world_size, + generator=self.generator) + return iter(indices) + + +class CustomBatchSampler(Sampler): + def __init__(self, batch_size, episode_len_l, sample_weights=None, replacement=True, eval=False, episode_first=True): + self.episode_len_l = episode_len_l + self.sample_weights = sample_weights + self.replacement = replacement + self.batch_size = batch_size + self.sample_probs = np.array(sample_weights) / np.sum(sample_weights) if sample_weights is not None else None + self.sum_dataset_len_l = np.cumsum([0] + [np.sum(episode_len) for episode_len in episode_len_l]) + self.max_steps = self.sum_dataset_len_l[-1] + self.episode_first = episode_first # 是否采用轨迹优先的采样策略 + if eval: + self.epochs = int(self.max_steps / batch_size) + else: + self.epochs = int(1e+10) + + def __iter__(self): + for _ in range(self.epochs): + batch = [] + for _ in range(self.batch_size): + if self.episode_first: + episode_idx = np.random.choice(len(self.episode_len_l), p=self.sample_probs) + step_idx = np.random.randint(self.sum_dataset_len_l[episode_idx], self.sum_dataset_len_l[episode_idx + 1]) + else: + # print("not episode_first") + step_idx = np.random.randint(self.sum_dataset_len_l[-1]) + batch.append(step_idx) + yield step_idx + #indices = torch.randperm(self.max_steps, generator=None) + #indices = indices.cpu().numpy() + + # return iter(indices) + +def _is_peft_model(model): + if is_peft_available(): + classes_to_check = (PeftModel,) if is_peft_available() else () + # Here we also check if the model is an instance of `PeftMixedModel` introduced in peft>=0.7.0: https://github.com/huggingface/transformers/pull/28321 + if version.parse(importlib.metadata.version("peft")) >= version.parse("0.7.0"): + from peft import PeftMixedModel + + classes_to_check = (*classes_to_check, PeftMixedModel) + return isinstance(model, classes_to_check) + return False +class DexVLATrainer(Trainer): + + def __init__(self, sampler_params, prefetch_factor=0, *args, **kwargs): + self.sampler_params = sampler_params + self.prefetch_factor = prefetch_factor + self.lora_module = kwargs['args'].lora_module + self.lang_type = 'model' if 'phi' in kwargs['model'].config.architectures[0].lower() else 'gpt_neox' + self.using_ema = getattr(kwargs['args'], "using_ema", False) + self.local_rank = kwargs['args'].local_rank + self.resume_from_checkpoint = kwargs['args'].resume_from_checkpoint + if self.using_ema: + if self.local_rank == 0: + print(">>>>>>>>>>>>>>>>>>>>>>>>>>Model weights is updated by EMA.<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<") + self.ema = EMAModel(model=kwargs['model'], power=0.75) + if self.resume_from_checkpoint: + if self.local_rank == 0: + print("Loading EMA weights from previous checkpoint...") + ckpt_dirs = glob.glob(os.path.join(kwargs['args'].output_dir, "checkpoint-*")) + ckpt_dirs = sorted(ckpt_dirs, key=lambda x: int(x.split("-")[-1])) + ema_state_dict = torch.load(os.path.join(kwargs['args'].output_dir, ckpt_dirs[-1], "ema_weights.pth"), map_location='cpu') + self.ema.averaged_model.load_state_dict(ema_state_dict, strict=True) + self.ema.optimization_step = int(ckpt_dirs[-1].split("-")[-1]) + + # print(os.environ.get("RANK", -1), kwargs['args'].local_rank) + + super().__init__(*args, **kwargs) + + def get_train_dataloader(self) -> DataLoader: + if self.train_dataset is None: + raise ValueError("Trainer: training requires a train_dataset.") + + train_dataset = self.train_dataset + data_collator = self.data_collator + + data_collator = self._get_collator_with_removed_columns(data_collator, description="training") + + dataloader_params = { + "batch_size": self._train_batch_size, + "collate_fn": data_collator, + "num_workers": self.args.dataloader_num_workers, + "pin_memory": self.args.dataloader_pin_memory, + "persistent_workers": self.args.dataloader_persistent_workers, + } + from transformers.trainer_utils import seed_worker + if not isinstance(train_dataset, torch.utils.data.IterableDataset): + # dataloader_params["sampler"] = CustomBatchSampler(**self.sampler_params['train'], eval=False) + dataloader_params["drop_last"] = self.args.dataloader_drop_last + dataloader_params["worker_init_fn"] = seed_worker + dataloader_params["shuffle"] = True + # dataloader_params['prefetch_factor'] = self.prefetch_factor + return self.accelerator.prepare(DataLoader(train_dataset, **dataloader_params)) + + def get_eval_dataloader(self, eval_dataset: Optional[Dataset] = None) -> DataLoader: + if eval_dataset is None and self.eval_dataset is None: + raise ValueError("Trainer: evaluation requires an eval_dataset.") + eval_dataset = eval_dataset if eval_dataset is not None else self.eval_dataset + data_collator = self.data_collator + + data_collator = self._get_collator_with_removed_columns(data_collator, description="evaluation") + + dataloader_params = { + "batch_size": self.args.eval_batch_size, + "collate_fn": data_collator, + "num_workers": self.args.dataloader_num_workers, + "pin_memory": self.args.dataloader_pin_memory, + "persistent_workers": self.args.dataloader_persistent_workers, + } + + if not isinstance(eval_dataset, torch.utils.data.IterableDataset): + # dataloader_params["sampler"] = CustomBatchSampler(**self.sampler_params['eval'], eval=True) + dataloader_params["shuffle"] = True + + dataloader_params["drop_last"] = self.args.dataloader_drop_last + + return self.accelerator.prepare(DataLoader(eval_dataset, **dataloader_params)) + + def _get_train_sampler(self) -> Optional[torch.utils.data.Sampler]: + if self.train_dataset is None or not has_length(self.train_dataset): + return None + + if self.args.group_by_modality_length: + lengths = self.train_dataset.modality_lengths + return LengthGroupedSampler( + # self.args.train_batch_size * self.args.gradient_accumulation_steps, # TODO: seems that we should not have gradient_accumulation_steps + self.args.train_batch_size, + world_size=self.args.world_size, + lengths=lengths, + group_by_modality=True, + ) + else: + return super()._get_train_sampler() + + def create_optimizer(self): + """ + Setup the optimizer. + + We provide a reasonable default that works well. If you want to use something else, you can pass a tuple in the + Trainer's init through `optimizers`, or subclass and override this method in a subclass. + """ + if is_sagemaker_mp_enabled(): + return super().create_optimizer() + + opt_model = self.model + + if self.optimizer is None: + non_lora_modules = ['vision_resampler', 'merger', 'lm_head', 'proj_to_action', 'text_hidden_fcs', + 'external_vit', 'input_action_proj', 'gt_action_proj', 'gt_film', 'reasoning_action_proj', + 'reasoning_film', 'channel_proj', 'xattn'] + if 'di_head' not in self.lora_module: + non_lora_modules.append('policy_head') + else: + non_lora_modules.append("x_embedder") + non_lora_modules.append("cond_obs_emb") + non_lora_modules.append("norm_after_pool") + decay_parameters = get_parameter_names(opt_model, ALL_LAYERNORM_LAYERS) + decay_parameters = [name for name in decay_parameters if "bias" not in name] + if self.args.non_lora_lr is not None: + # non_lora_parameters = [name for name, _ in opt_model.named_parameters() if ("mm_projector" in name or "vision_tower" in name)] + non_lora_parameters = [] + test = [] + for name, module in opt_model.named_parameters(): + + # if 'layers' in name and 'vision' not in name and 'gpt_neox' in name: # gptneoxl LLM的参数 + if 'policy_head' not in name and 'layers' in name and 'vision' not in name and self.lang_type in name: # gptneoxl LLM的参数 + if 'llm' not in self.lora_module: + non_lora_parameters.append(name) + pass + + elif any(key in name for key in non_lora_modules): # vision adapter、action head的参数 + # non_lora_parameters.append(name) + non_lora_parameters.append(name) + + optimizer_grouped_parameters = [ + { + "params": [ + p for n, p in opt_model.named_parameters() if + (n in decay_parameters and n not in non_lora_parameters and p.requires_grad) # lora and decay + ], + "weight_decay": self.args.weight_decay, + }, + { + "params": [ + p for n, p in opt_model.named_parameters() if + (n not in decay_parameters and n not in non_lora_parameters and p.requires_grad) # lora and non-decay + ], + "weight_decay": 0.0, + }, + { + "params": [ + p for n, p in opt_model.named_parameters() if + (n in decay_parameters and n in non_lora_parameters and p.requires_grad) # non-lora and decay + ], + "weight_decay": self.args.weight_decay, + "lr": self.args.non_lora_lr, + }, + { + "params": [ + p for n, p in opt_model.named_parameters() if + (n not in decay_parameters and n in non_lora_parameters and p.requires_grad) # non-lora and non-decay + ], + "weight_decay": 0.0, + "lr": self.args.non_lora_lr, + }, + ] + assert len(optimizer_grouped_parameters[1][ + 'params']) == 0, f"{optimizer_grouped_parameters[1]['params']} should be empty!!!!!" + else: + optimizer_grouped_parameters = [ + { + "params": [ + p for n, p in opt_model.named_parameters() if (n in decay_parameters and p.requires_grad) + ], + "weight_decay": self.args.weight_decay, + }, + { + "params": [ + p for n, p in opt_model.named_parameters() if + (n not in decay_parameters and p.requires_grad) + ], + "weight_decay": 0.0, + }, + ] + # for each in optimizer_grouped_parameters: + + optimizer_cls, optimizer_kwargs = Trainer.get_optimizer_cls_and_kwargs(self.args) + + self.optimizer = optimizer_cls(optimizer_grouped_parameters, **optimizer_kwargs) + if optimizer_cls.__name__ == "Adam8bit": + import bitsandbytes + + manager = bitsandbytes.optim.GlobalOptimManager.get_instance() + + skipped = 0 + for module in opt_model.modules(): + if isinstance(module, nn.Embedding): + skipped += sum({p.data_ptr(): p.numel() for p in module.parameters()}.values()) + logger.info(f"skipped {module}: {skipped / 2 ** 20}M params") + manager.register_module_override(module, "weight", {"optim_bits": 32}) + logger.debug(f"bitsandbytes: will optimize {module} in fp32") + logger.info(f"skipped: {skipped / 2 ** 20}M params") + + return self.optimizer + + def training_step(self, model: nn.Module, inputs: Dict[str, Union[torch.Tensor, Any]]) -> torch.Tensor: + """ + Perform a training step on a batch of inputs. + + Subclass and override to inject custom behavior. + + Args: + model (`nn.Module`): + The model to train. + inputs (`Dict[str, Union[torch.Tensor, Any]]`): + The inputs and targets of the model. + + The dictionary will be unpacked before being fed to the model. Most models expect the targets under the + argument `labels`. Check your model's documentation for all accepted arguments. + + Return: + `torch.Tensor`: The tensor with training loss on this batch. + """ + model.train() + inputs = self._prepare_inputs(inputs) + + if is_sagemaker_mp_enabled(): + loss_mb = smp_forward_backward(model, inputs, self.args.gradient_accumulation_steps) + return loss_mb.reduce_mean().detach().to(self.args.device) + + with self.compute_loss_context_manager(): + ###############################modified################################## + # print("#####this is input#######################") + # print('inputs:', inputs) + loss = self.compute_loss(model, inputs, return_outputs=False) # change return_outputs to True + + ######################################################################### + + if self.args.n_gpu > 1: + loss = loss.mean() # mean() to average on multi-gpu parallel training + + ###############################modified################################## + if self.use_apex: + with amp.scale_loss(loss, self.optimizer) as scaled_loss: + scaled_loss.backward() + else: + self.accelerator.backward(loss['loss']) # modified + loss = {k:v.detach() for k,v in loss.items()} # modified + + return loss['loss'] / self.args.gradient_accumulation_steps, loss # modified + ####################################################################### + + + + # modified from transformers.trainer.Trainer, only change the metric record + def _inner_training_loop( + self, batch_size=None, args=None, resume_from_checkpoint=None, trial=None, ignore_keys_for_eval=None + ): + self.accelerator.free_memory() + self._train_batch_size = batch_size + if self.args.auto_find_batch_size: + if self.state.train_batch_size != self._train_batch_size: + from accelerate.utils import release_memory + + (self.model_wrapped,) = release_memory(self.model_wrapped) + self.model_wrapped = self.model + + # Check for DeepSpeed *after* the intial pass and modify the config + if self.is_deepspeed_enabled: + # Temporarily unset `self.args.train_batch_size` + original_bs = self.args.per_device_train_batch_size + self.args.per_device_train_batch_size = self._train_batch_size // max(1, self.args.n_gpu) + self.propagate_args_to_deepspeed(True) + self.args.per_device_train_batch_size = original_bs + self.state.train_batch_size = self._train_batch_size + logger.debug(f"Currently training with a batch size of: {self._train_batch_size}") + # Data loader and number of training steps + train_dataloader = self.get_train_dataloader() + if self.is_fsdp_xla_v2_enabled: + train_dataloader = tpu_spmd_dataloader(train_dataloader) + + # Setting up training control variables: + # number of training epochs: num_train_epochs + # number of training steps per epoch: num_update_steps_per_epoch + # total number of training steps to execute: max_steps + total_train_batch_size = self._train_batch_size * args.gradient_accumulation_steps * args.world_size + + len_dataloader = None + num_train_tokens = None + if has_length(train_dataloader): + len_dataloader = len(train_dataloader) + num_update_steps_per_epoch = len_dataloader // args.gradient_accumulation_steps + num_update_steps_per_epoch = max(num_update_steps_per_epoch, 1) + num_examples = self.num_examples(train_dataloader) + if args.max_steps > 0: + max_steps = args.max_steps + num_train_epochs = args.max_steps // num_update_steps_per_epoch + int( + args.max_steps % num_update_steps_per_epoch > 0 + ) + # May be slightly incorrect if the last batch in the training dataloader has a smaller size but it's + # the best we can do. + num_train_samples = args.max_steps * total_train_batch_size + if args.include_tokens_per_second: + num_train_tokens = ( + self.num_tokens(train_dataloader, args.max_steps) * args.gradient_accumulation_steps + ) + else: + max_steps = math.ceil(args.num_train_epochs * num_update_steps_per_epoch) + num_train_epochs = math.ceil(args.num_train_epochs) + num_train_samples = self.num_examples(train_dataloader) * args.num_train_epochs + if args.include_tokens_per_second: + num_train_tokens = self.num_tokens(train_dataloader) * args.num_train_epochs + elif args.max_steps > 0: # Rely on max_steps when dataloader does not have a working size + max_steps = args.max_steps + # Setting a very large number of epochs so we go as many times as necessary over the iterator. + num_train_epochs = sys.maxsize + num_update_steps_per_epoch = max_steps + num_examples = total_train_batch_size * args.max_steps + num_train_samples = args.max_steps * total_train_batch_size + if args.include_tokens_per_second: + num_train_tokens = self.num_tokens(train_dataloader, args.max_steps) * args.gradient_accumulation_steps + else: + raise ValueError( + "args.max_steps must be set to a positive value if dataloader does not have a length, was" + f" {args.max_steps}" + ) + + if DebugOption.UNDERFLOW_OVERFLOW in self.args.debug: + if self.args.n_gpu > 1: + # nn.DataParallel(model) replicates the model, creating new variables and module + # references registered here no longer work on other gpus, breaking the module + raise ValueError( + "Currently --debug underflow_overflow is not supported under DP. Please use DDP" + " (torchrun or torch.distributed.launch (deprecated))." + ) + else: + debug_overflow = DebugUnderflowOverflow(self.model) # noqa + + delay_optimizer_creation = is_sagemaker_mp_enabled() or self.is_fsdp_xla_enabled or self.is_fsdp_enabled + + # We need to reset the scheduler, as its parameters may be different on subsequent calls + if self._created_lr_scheduler: + self.lr_scheduler = None + self._created_lr_scheduler = False + + if self.is_deepspeed_enabled: + self.optimizer, self.lr_scheduler = deepspeed_init(self, num_training_steps=max_steps) + + if not delay_optimizer_creation: + self.create_optimizer_and_scheduler(num_training_steps=max_steps) + + self.state = TrainerState( + stateful_callbacks=[ + cb for cb in self.callback_handler.callbacks + [self.control] if isinstance(cb, ExportableState) + ] + ) + self.state.is_hyper_param_search = trial is not None + self.state.train_batch_size = self._train_batch_size + + # Compute absolute values for logging, eval, and save if given as ratio + if args.logging_steps is not None: + if args.logging_steps < 1: + self.state.logging_steps = math.ceil(max_steps * args.logging_steps) + else: + self.state.logging_steps = args.logging_steps + if args.eval_steps is not None: + if args.eval_steps < 1: + self.state.eval_steps = math.ceil(max_steps * args.eval_steps) + else: + self.state.eval_steps = args.eval_steps + if args.save_steps is not None: + if args.save_steps < 1: + self.state.save_steps = math.ceil(max_steps * args.save_steps) + else: + self.state.save_steps = args.save_steps + + # Activate gradient checkpointing if needed + if args.gradient_checkpointing: + self.model.gradient_checkpointing_enable(gradient_checkpointing_kwargs=args.gradient_checkpointing_kwargs) + + model = self._wrap_model(self.model_wrapped) + + # as the model is wrapped, don't use `accelerator.prepare` + # this is for unhandled cases such as + # FSDP-XLA, SageMaker MP/DP, DataParallel, IPEX + use_accelerator_prepare = True if model is self.model else False + + if delay_optimizer_creation: + if use_accelerator_prepare: + self._fsdp_qlora_plugin_updates() + self.model = self.accelerator.prepare(self.model) + self.create_optimizer_and_scheduler(num_training_steps=max_steps) + + # prepare using `accelerator` prepare + if use_accelerator_prepare: + self.model.train() + if hasattr(self.lr_scheduler, "step"): + if self.use_apex: + model = self.accelerator.prepare(self.model) + else: + model, self.optimizer = self.accelerator.prepare(self.model, self.optimizer) + else: + # to handle cases wherein we pass "DummyScheduler" such as when it is specified in DeepSpeed config. + model, self.optimizer, self.lr_scheduler = self.accelerator.prepare( + self.model, self.optimizer, self.lr_scheduler + ) + elif self.args.optim in [OptimizerNames.LOMO, OptimizerNames.ADALOMO]: + # In this case we are in DDP + LOMO, which should be supported + self.optimizer = self.accelerator.prepare(self.optimizer) + + if self.is_fsdp_enabled: + self.model = self.model_wrapped = model + + # for the rest of this function `model` is the outside model, whether it was wrapped or not + if model is not self.model: + self.model_wrapped = model + + # backward compatibility + if self.is_deepspeed_enabled: + self.deepspeed = self.model_wrapped + + # ckpt loading + if resume_from_checkpoint is not None: + if self.is_deepspeed_enabled: + deepspeed_load_checkpoint( + self.model_wrapped, resume_from_checkpoint, load_module_strict=not _is_peft_model(self.model) + ) + elif is_sagemaker_mp_enabled() or self.is_fsdp_enabled: + self._load_from_checkpoint(resume_from_checkpoint, self.model_wrapped) + + # Check if saved optimizer or scheduler states exist + self._load_optimizer_and_scheduler(resume_from_checkpoint) + + # important: at this point: + # self.model is the Transformers Model + # self.model_wrapped is DDP(Transformers Model), Deepspeed(Transformers Model), + # FSDP(Transformers Model), Dynamo Optimized Module(Transformers Model) etc. + + # Train! + logger.info("***** Running training *****") + logger.info(f" Num examples = {num_examples:,}") + logger.info(f" Num Epochs = {num_train_epochs:,}") + logger.info(f" Instantaneous batch size per device = {self.args.per_device_train_batch_size:,}") + if self.args.per_device_train_batch_size != self._train_batch_size: + logger.info(f" Training with DataParallel so batch size has been adjusted to: {self._train_batch_size:,}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_train_batch_size:,}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {max_steps:,}") + logger.info(f" Number of trainable parameters = {get_model_param_count(model, trainable_only=True):,}") + + self.state.epoch = 0 + start_time = time.time() + epochs_trained = 0 + steps_trained_in_current_epoch = 0 + steps_trained_progress_bar = None + + # Check if continuing training from a checkpoint + if resume_from_checkpoint is not None and os.path.isfile( + os.path.join(resume_from_checkpoint, TRAINER_STATE_NAME) + ): + self.state = TrainerState.load_from_json(os.path.join(resume_from_checkpoint, TRAINER_STATE_NAME)) + self.compare_trainer_and_checkpoint_args(self.args, self.state) + self._load_callback_state() + epochs_trained = int(self.state.global_step // num_update_steps_per_epoch) + if not args.ignore_data_skip: + steps_trained_in_current_epoch = self.state.global_step % (num_update_steps_per_epoch) + steps_trained_in_current_epoch *= args.gradient_accumulation_steps + else: + steps_trained_in_current_epoch = 0 + + logger.info(" Continuing training from checkpoint, will skip to saved global_step") + logger.info(f" Continuing training from epoch {epochs_trained}") + logger.info(f" Continuing training from global step {self.state.global_step}") + if not args.ignore_data_skip: + logger.info( + f" Will skip the first {epochs_trained} epochs then the first" + f" {steps_trained_in_current_epoch} batches in the first epoch." + ) + + # Update the references + self.callback_handler.model = self.model + self.callback_handler.optimizer = self.optimizer + self.callback_handler.lr_scheduler = self.lr_scheduler + self.callback_handler.train_dataloader = train_dataloader + if self.hp_name is not None and self._trial is not None: + # use self._trial because the SigOpt/Optuna hpo only call `_hp_search_setup(trial)` instead of passing trial + # parameter to Train when using DDP. + self.state.trial_name = self.hp_name(self._trial) + if trial is not None: + assignments = trial.assignments if self.hp_search_backend == HPSearchBackend.SIGOPT else trial + self.state.trial_params = hp_params(assignments) + else: + self.state.trial_params = None + # This should be the same if the state has been saved but in case the training arguments changed, it's safer + # to set this after the load. + self.state.max_steps = max_steps + self.state.num_train_epochs = num_train_epochs + self.state.is_local_process_zero = self.is_local_process_zero() + self.state.is_world_process_zero = self.is_world_process_zero() + + # tr_loss is a tensor to avoid synchronization of TPUs through .item() + tr_loss = torch.tensor(0.0).to(args.device) + ################################################################################## + custom_loss = { + 'llm_loss': torch.tensor(0.0).to(args.device), + 'action_loss': torch.tensor(0.0).to(args.device), + } + ################################################################################## + # _total_loss_scalar is updated everytime .item() has to be called on tr_loss and stores the sum of all losses + self._total_loss_scalar = 0.0 + self._globalstep_last_logged = self.state.global_step + model.zero_grad() + grad_norm: Optional[float] = None + self.control = self.callback_handler.on_train_begin(args, self.state, self.control) + + if args.eval_on_start: + self._evaluate(trial, ignore_keys_for_eval, skip_scheduler=True) + + total_batched_samples = 0 + for epoch in range(epochs_trained, num_train_epochs): + epoch_iterator = train_dataloader + if hasattr(epoch_iterator, "set_epoch"): + epoch_iterator.set_epoch(epoch) + + # Reset the past mems state at the beginning of each epoch if necessary. + if args.past_index >= 0: + self._past = None + + steps_in_epoch = ( + len(epoch_iterator) + if len_dataloader is not None + else args.max_steps * args.gradient_accumulation_steps + ) + self.control = self.callback_handler.on_epoch_begin(args, self.state, self.control) + + if epoch == epochs_trained and resume_from_checkpoint is not None and steps_trained_in_current_epoch == 0: + self._load_rng_state(resume_from_checkpoint) + + rng_to_sync = False + steps_skipped = 0 + if steps_trained_in_current_epoch > 0: + epoch_iterator = skip_first_batches(epoch_iterator, steps_trained_in_current_epoch) + steps_skipped = steps_trained_in_current_epoch + steps_trained_in_current_epoch = 0 + rng_to_sync = True + + step = -1 + for step, inputs in enumerate(epoch_iterator): + total_batched_samples += 1 + + if self.args.include_num_input_tokens_seen: + main_input_name = getattr(self.model, "main_input_name", "input_ids") + if main_input_name not in inputs: + logger.warning( + "Tried to track the number of tokens seen, however the current model is " + "not configured properly to know what item is the input. To fix this, add " + "a `main_input_name` attribute to the model class you are using." + ) + else: + self.state.num_input_tokens_seen += ( + torch.sum( + self.accelerator.gather( + torch.tensor( + inputs[main_input_name].numel(), device=self.args.device, dtype=torch.int64 + ) + ) + ) + .cpu() + .item() + ) + if rng_to_sync: + self._load_rng_state(resume_from_checkpoint) + rng_to_sync = False + + # Skip past any already trained steps if resuming training + if steps_trained_in_current_epoch > 0: + steps_trained_in_current_epoch -= 1 + if steps_trained_progress_bar is not None: + steps_trained_progress_bar.update(1) + if steps_trained_in_current_epoch == 0: + self._load_rng_state(resume_from_checkpoint) + continue + elif steps_trained_progress_bar is not None: + steps_trained_progress_bar.close() + steps_trained_progress_bar = None + + if step % args.gradient_accumulation_steps == 0: + self.control = self.callback_handler.on_step_begin(args, self.state, self.control) + #################################################### + with self.accelerator.accumulate(model): + tr_loss_step, all_loss = self.training_step(model, inputs) # modified,return all_loss + #################################################### + + if ( + args.logging_nan_inf_filter + and not is_torch_xla_available() + and (torch.isnan(tr_loss_step) or torch.isinf(tr_loss_step)) + ): + # if loss is nan or inf simply add the average of previous logged losses + tr_loss += tr_loss / (1 + self.state.global_step - self._globalstep_last_logged) + ##################################################################################################### + for k,v in all_loss.items(): + if k == 'loss': + continue + custom_loss[k] += all_loss[k] / (1 + self.state.global_step - self._globalstep_last_logged) + ##################################################################################################### + else: + if tr_loss.device != tr_loss_step.device: + raise ValueError( + f"Calculated loss must be on the original device: {tr_loss.device} but device in use is {tr_loss_step.device}" + ) + tr_loss += tr_loss_step + ################################################################### + for k, v in all_loss.items(): + if k == 'loss': + continue + custom_loss[k] += v + ################################################################### + + self.current_flos += float(self.floating_point_ops(inputs)) + + is_last_step_and_steps_less_than_grad_acc = ( + steps_in_epoch <= args.gradient_accumulation_steps and (step + 1) == steps_in_epoch + ) + + if ( + total_batched_samples % args.gradient_accumulation_steps == 0 + or + # last step in epoch but step is always smaller than gradient_accumulation_steps + is_last_step_and_steps_less_than_grad_acc + ): + # the `or` condition of `is_last_step_and_steps_less_than_grad_acc` is not covered + # in accelerate. So, explicitly enable sync gradients to True in that case. + if is_last_step_and_steps_less_than_grad_acc: + self.accelerator.gradient_state._set_sync_gradients(True) + + # Gradient clipping + if args.max_grad_norm is not None and args.max_grad_norm > 0: + # deepspeed does its own clipping + + if is_sagemaker_mp_enabled() and args.fp16: + _grad_norm = self.optimizer.clip_master_grads(args.max_grad_norm) + elif self.use_apex: + # Revert to normal clipping otherwise, handling Apex or full precision + _grad_norm = nn.utils.clip_grad_norm_( + amp.master_params(self.optimizer), + args.max_grad_norm, + ) + else: + _grad_norm = self.accelerator.clip_grad_norm_( + model.parameters(), + args.max_grad_norm, + ) + + if ( + is_accelerate_available() + and self.accelerator.distributed_type == DistributedType.DEEPSPEED + ): + grad_norm = model.get_global_grad_norm() + # In some cases the grad norm may not return a float + if hasattr(grad_norm, "item"): + grad_norm = grad_norm.item() + else: + grad_norm = _grad_norm + + self.control = self.callback_handler.on_pre_optimizer_step(args, self.state, self.control) + + self.optimizer.step() + self.control = self.callback_handler.on_optimizer_step(args, self.state, self.control) + + optimizer_was_run = not self.accelerator.optimizer_step_was_skipped + if optimizer_was_run: + # Delay optimizer scheduling until metrics are generated + if not isinstance(self.lr_scheduler, torch.optim.lr_scheduler.ReduceLROnPlateau): + self.lr_scheduler.step() + + model.zero_grad() + ################################################################### + if self.using_ema: + self.ema.step(model) + ################################################################### + self.state.global_step += 1 + self.state.epoch = epoch + (step + 1 + steps_skipped) / steps_in_epoch + self.control = self.callback_handler.on_step_end(args, self.state, self.control) + ############################################################################################################################################### + # self._maybe_log_save_evaluate(tr_loss, grad_norm, model, trial, epoch, ignore_keys_for_eval) + self._maybe_log_save_evaluate(tr_loss, model, trial, epoch, ignore_keys_for_eval, all_loss=custom_loss) + ############################################################################################################################################### + else: + self.control = self.callback_handler.on_substep_end(args, self.state, self.control) + + if self.control.should_epoch_stop or self.control.should_training_stop: + # PyTorch/XLA relies on the data loader to insert the mark_step for + # each step. Since we are breaking the loop early, we need to manually + # insert the mark_step here. + if is_torch_xla_available(): + xm.mark_step() + break + if step < 0: + logger.warning( + "There seems not to be a single sample in your epoch_iterator, stopping training at step" + f" {self.state.global_step}! This is expected if you're using an IterableDataset and set" + f" num_steps ({max_steps}) higher than the number of available samples." + ) + self.control.should_training_stop = True + + self.control = self.callback_handler.on_epoch_end(args, self.state, self.control) + ############################################################################################################################################### + # self._maybe_log_save_evaluate(tr_loss, model, trial, epoch, ignore_keys_for_eval) + self._maybe_log_save_evaluate(tr_loss, model, trial, epoch, ignore_keys_for_eval, all_loss=custom_loss) + ############################################################################################################################################### + + if DebugOption.TPU_METRICS_DEBUG in self.args.debug: + if is_torch_xla_available(): + # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.) + xm.master_print(met.metrics_report()) + else: + logger.warning( + "You enabled PyTorch/XLA debug metrics but you don't have a TPU " + "configured. Check your training configuration if this is unexpected." + ) + if self.control.should_training_stop: + break + + if args.past_index and hasattr(self, "_past"): + # Clean the state at the end of training + delattr(self, "_past") + + logger.info("\n\nTraining completed. Do not forget to share your model on huggingface.co/models =)\n\n") + if args.load_best_model_at_end and self.state.best_model_checkpoint is not None: + # Wait for everyone to get here so we are sure the model has been saved by process 0. + if is_torch_xla_available(): + xm.rendezvous("load_best_model_at_end") + elif args.parallel_mode == ParallelMode.DISTRIBUTED: + dist.barrier() + elif is_sagemaker_mp_enabled(): + smp.barrier() + + self._load_best_model() + + # add remaining tr_loss + self._total_loss_scalar += tr_loss.item() + effective_global_step = max(self.state.global_step, 0.001) # Avoid ZeroDivisionError + train_loss = self._total_loss_scalar / effective_global_step + + metrics = speed_metrics( + "train", + start_time, + num_samples=num_train_samples, + num_steps=self.state.max_steps, + num_tokens=num_train_tokens, + ) + self.store_flos() + metrics["total_flos"] = self.state.total_flos + metrics["train_loss"] = train_loss + + self.is_in_train = False + + self._memory_tracker.stop_and_update_metrics(metrics) + + self.log(metrics) + + run_dir = self._get_output_dir(trial) + checkpoints_sorted = self._sorted_checkpoints(use_mtime=False, output_dir=run_dir) + + # Delete the last checkpoint when save_total_limit=1 if it's different from the best checkpoint and process allowed to save. + if self.args.should_save and self.state.best_model_checkpoint is not None and self.args.save_total_limit == 1: + for checkpoint in checkpoints_sorted: + if not os.path.samefile(checkpoint, self.state.best_model_checkpoint): + logger.info(f"Deleting older checkpoint [{checkpoint}] due to args.save_total_limit") + shutil.rmtree(checkpoint, ignore_errors=True) + + self.control = self.callback_handler.on_train_end(args, self.state, self.control) + + # Wait for the checkpoint to be uploaded. + self._finish_current_push() + + # After training we make sure to retrieve back the original forward pass method + # for the embedding layer by removing the forward post hook. + if self.neftune_noise_alpha is not None: + self._deactivate_neftune(self.model) + + return TrainOutput(self.state.global_step, train_loss, metrics) + + def _maybe_log_save_evaluate(self, tr_loss, model, trial, epoch, ignore_keys_for_eval, all_loss=None): + if self.control.should_log and self.state.global_step > self._globalstep_last_logged: + if is_torch_tpu_available(): + xm.mark_step() + + logs: Dict[str, float] = {} + + # all_gather + mean() to get average loss over all processes + tr_loss_scalar = self._nested_gather(tr_loss).mean().item() + + #################################################modified####################################################### + custom_loss = { + 'llm_loss': torch.tensor(0.0).to(tr_loss.device), + 'action_loss': torch.tensor(0.0).to(tr_loss.device), + } + for k,v in all_loss.items(): + if k == 'loss': + continue + custom_loss[k] = self._nested_gather(v).mean().item() + ################################################################################################################ + + # reset tr_loss to zero + tr_loss -= tr_loss + ####################modified##################### + for k,v in all_loss.items(): + if k == 'loss': + continue + all_loss[k] -= all_loss[k] + ################################################## + + logs["loss"] = round(tr_loss_scalar / (self.state.global_step - self._globalstep_last_logged), 4) + logs["learning_rate"] = self._get_learning_rate() + ##############################################modified######################################################## + for k,v in custom_loss.items(): + if k == 'loss': + continue + logs[k] = round(v / (self.state.global_step - self._globalstep_last_logged), 4) + ################################################################################################################ + + self._total_loss_scalar += tr_loss_scalar + self._globalstep_last_logged = self.state.global_step + self.store_flos() + + self.log(logs) + + metrics = None + if self.control.should_evaluate: + metrics = self.evaluate(ignore_keys=ignore_keys_for_eval) + self._report_to_hp_search(trial, self.state.global_step, metrics) + + # Run delayed LR scheduler now that metrics are populated + if isinstance(self.lr_scheduler, torch.optim.lr_scheduler.ReduceLROnPlateau): + metric_to_check = self.args.metric_for_best_model + if not metric_to_check.startswith("eval_"): + metric_to_check = f"eval_{metric_to_check}" + self.lr_scheduler.step(metrics[metric_to_check]) + + if self.control.should_save: + ##############################################modified######################################################## + if self.using_ema: + checkpoint_folder = f"{PREFIX_CHECKPOINT_DIR}-{self.state.global_step}" + run_dir = self._get_output_dir(trial=trial) + output_dir = os.path.join(run_dir, checkpoint_folder) + os.makedirs(output_dir, exist_ok=True) + # if not os.path.isfile(os.path.join(output_dir, "ema_weights.pth")): + if self.local_rank == 0: + ema_state_dict = self.ema.averaged_model.state_dict() + # self._save_checkpoint(model, trial, metrics=metrics, using_ema=True) + print(f"-----------------------------Saving EMA Weights on {self.local_rank}-----------------------------") + torch.save(ema_state_dict, os.path.join(output_dir, "ema_weights.pth")) + self._save_checkpoint(model, trial, metrics=metrics, using_ema=False) + ############################################################################################################## + self.control = self.callback_handler.on_save(self.args, self.state, self.control) + + def _load_from_checkpoint(self, resume_from_checkpoint, model=None): + if model is None: + model = self.model + + config_file = os.path.join(resume_from_checkpoint, CONFIG_NAME) + adapter_weights_file = os.path.join(resume_from_checkpoint, ADAPTER_WEIGHTS_NAME) + adapter_safe_weights_file = os.path.join(resume_from_checkpoint, ADAPTER_SAFE_WEIGHTS_NAME) + weights_file = os.path.join(resume_from_checkpoint, WEIGHTS_NAME) + weights_index_file = os.path.join(resume_from_checkpoint, WEIGHTS_INDEX_NAME) + safe_weights_file = os.path.join(resume_from_checkpoint, SAFE_WEIGHTS_NAME) + safe_weights_index_file = os.path.join(resume_from_checkpoint, SAFE_WEIGHTS_INDEX_NAME) + is_fsdp_ckpt = os.path.isdir(resume_from_checkpoint) and ( + # this checks the FSDP state dict when `SHARDED_STATE_DICT` is used + any( + FSDP_MODEL_NAME in folder_name + for folder_name in os.listdir(resume_from_checkpoint) + if os.path.isdir(os.path.join(resume_from_checkpoint, folder_name)) + ) + # this checks the FSDP state dict when `FULL_STATE_DICT` is used + or os.path.isfile(os.path.join(resume_from_checkpoint, f"{FSDP_MODEL_NAME}.bin")) + ) + # if multiple adapters exist, they get saved in sub directories + adapter_subdirs = ( + [ + folder_name + for folder_name in os.listdir(resume_from_checkpoint) + if os.path.isdir(os.path.join(resume_from_checkpoint, folder_name)) + and ( + os.path.isfile(os.path.join(resume_from_checkpoint, folder_name, ADAPTER_WEIGHTS_NAME)) + or os.path.isfile(os.path.join(resume_from_checkpoint, folder_name, ADAPTER_SAFE_WEIGHTS_NAME)) + ) + ] + if os.path.isdir(resume_from_checkpoint) + else [] + ) + + if is_fsdp_ckpt and not self.is_fsdp_enabled: + raise ValueError(f"Checkpoint found at {resume_from_checkpoint} is only supported when using PyTorch FSDP") + + if not ( + any( + os.path.isfile(f) + for f in [ + weights_file, + safe_weights_file, + weights_index_file, + safe_weights_index_file, + adapter_weights_file, + adapter_safe_weights_file, + ] + ) + or is_fsdp_ckpt + or adapter_subdirs + ): + raise ValueError(f"Can't find a valid checkpoint at {resume_from_checkpoint}") + + logger.info(f"Loading model from {resume_from_checkpoint}.") + + if os.path.isfile(config_file): + config = PretrainedConfig.from_json_file(config_file) + checkpoint_version = config.transformers_version + if checkpoint_version is not None and checkpoint_version != __version__: + logger.warning( + f"You are resuming training from a checkpoint trained with {checkpoint_version} of " + f"Transformers but your current version is {__version__}. This is not recommended and could " + "yield to errors or unwanted behaviors." + ) + + if os.path.isfile(weights_file) or os.path.isfile(safe_weights_file) or is_fsdp_ckpt: + weights_only_kwarg = {"weights_only": True} if is_torch_greater_or_equal_than_1_13 else {} + # If the model is on the GPU, it still works! + if is_sagemaker_mp_enabled(): + if os.path.isfile(os.path.join(resume_from_checkpoint, "user_content.pt")): + # If the 'user_content.pt' file exists, load with the new smp api. + # Checkpoint must have been saved with the new smp api. + smp.resume_from_checkpoint( + path=resume_from_checkpoint, tag=WEIGHTS_NAME, partial=False, load_optimizer=False + ) + else: + # If the 'user_content.pt' file does NOT exist, load with the old smp api. + # Checkpoint must have been saved with the old smp api. + if hasattr(self.args, "fp16") and self.args.fp16 is True: + logger.warning( + "Enabling FP16 and loading from smp < 1.10 checkpoint together is not suppported." + ) + state_dict = torch.load( + weights_file, + map_location="cpu", + **weights_only_kwarg, + ) + # Required for smp to not auto-translate state_dict from hf to smp (is already smp). + state_dict["_smp_is_partial"] = False + load_result = model.load_state_dict(state_dict, strict=True) + # release memory + del state_dict + elif self.is_fsdp_enabled: + load_fsdp_model( + self.accelerator.state.fsdp_plugin, + self.accelerator, + model, + resume_from_checkpoint, + **_get_fsdp_ckpt_kwargs(), + ) + else: + # We load the model state dict on the CPU to avoid an OOM error. + if self.args.save_safetensors and os.path.isfile(safe_weights_file): + state_dict = safetensors.torch.load_file(safe_weights_file, device="cpu") + else: + state_dict = torch.load( + weights_file, + map_location="cpu", + **weights_only_kwarg, + ) + + # workaround for FSDP bug https://github.com/pytorch/pytorch/issues/82963 + # which takes *args instead of **kwargs + load_result = model.load_state_dict(state_dict, False) + # release memory + del state_dict + self._issue_warnings_after_load(load_result) + + # Load adapters following PR # 24096 + elif _is_peft_model(model): + # If train a model using PEFT & LoRA, assume that adapter have been saved properly. + if hasattr(model, "active_adapter") and hasattr(model, "load_adapter"): + if os.path.exists(resume_from_checkpoint): + model.load_adapter(resume_from_checkpoint, model.active_adapter, is_trainable=True) + else: + logger.warning( + "The intermediate checkpoints of PEFT may not be saved correctly, " + f"consider using a custom callback to save {ADAPTER_WEIGHTS_NAME} in corresponding saving folders. " + "Check some examples here: https://github.com/huggingface/peft/issues/96" + ) + else: + logger.warning("Could not load adapter model, make sure to have `peft>=0.3.0` installed") + else: + # We load the sharded checkpoint + load_result = load_sharded_checkpoint( + model, resume_from_checkpoint, strict=is_sagemaker_mp_enabled(), prefer_safe=self.args.save_safetensors + ) + if not is_sagemaker_mp_enabled(): + self._issue_warnings_after_load(load_result) + + def _save_checkpoint(self, model, trial, metrics=None, using_ema=False): + # In all cases, including ddp/dp/deepspeed, self.model is always a reference to the model we + # want to save except FullyShardedDDP. + # assert unwrap_model(model) is self.model, "internal model should be a reference to self.model" + + # Save model checkpoint + checkpoint_folder = f"{PREFIX_CHECKPOINT_DIR}-{self.state.global_step}" + + if self.hp_search_backend is None and trial is None: + self.store_flos() + + run_dir = self._get_output_dir(trial=trial) + if using_ema: + output_dir = os.path.join(run_dir, checkpoint_folder, 'ema') + else: + output_dir = os.path.join(run_dir, checkpoint_folder) + self.save_model(output_dir, _internal_call=True) + + if not self.args.save_only_model: + # Save optimizer and scheduler + self._save_optimizer_and_scheduler(output_dir) + # Save RNG state + self._save_rng_state(output_dir) + + # Determine the new best metric / best model checkpoint + if metrics is not None and self.args.metric_for_best_model is not None: + metric_to_check = self.args.metric_for_best_model + if not metric_to_check.startswith("eval_"): + metric_to_check = f"eval_{metric_to_check}" + try: + metric_value = metrics[metric_to_check] + except KeyError as exc: + raise KeyError( + f"The `metric_for_best_model` training argument is set to '{metric_to_check}', which is not found in the evaluation metrics. " + f"The available evaluation metrics are: {list(metrics.keys())}. Consider changing the `metric_for_best_model` via the TrainingArguments." + ) from exc + + operator = np.greater if self.args.greater_is_better else np.less + if ( + self.state.best_metric is None + or self.state.best_model_checkpoint is None + or operator(metric_value, self.state.best_metric) + ): + self.state.best_metric = metric_value + self.state.best_model_checkpoint = output_dir + + # Save the Trainer state + if self.args.should_save: + # Update `ExportableState` callbacks and `TrainerControl` state to where we are currently + for cb in [ + cb for cb in self.callback_handler.callbacks + [self.control] if isinstance(cb, ExportableState) + ]: + cb_name = cb.__class__.__name__ + cb_state = cb.state() + if isinstance(self.state.stateful_callbacks[cb_name], list): + self.state.stateful_callbacks[cb_name].append(cb_state) + else: + self.state.stateful_callbacks[cb_name] = cb_state + self.state.save_to_json(os.path.join(output_dir, TRAINER_STATE_NAME)) + + if self.args.push_to_hub: + self._push_from_checkpoint(output_dir) + + # Maybe delete some older checkpoints. + if self.args.should_save: + # Solely rely on numerical checkpoint id for rotation. + # mtime is not reliable especially on some fuse fs in cloud environments. + self._rotate_checkpoints(use_mtime=False, output_dir=run_dir) + + def _save(self, output_dir: Optional[str] = None, state_dict=None): + + # if 'ema' in output_dir.split('/')[-1]: + # print(f"-----------------------------Saving EMA Weights-----------------------------") + # ema_state_dict = self.ema.averaged_model.state_dict() + # # super(QWen2VLATrainer, self)._save(output_dir, ema_state_dict) + # os.makedirs(output_dir, exist_ok=True) + # torch.save(ema_state_dict, os.path.join(output_dir, "ema_weights.pth")) + # else: + # print("+++++++++++++++++++++++++++++Saving Normal Weights+++++++++++++++++++++++++++++") + super(DexVLATrainer, self)._save(output_dir, state_dict) + # If we are executing this function, we are the process zero, so we don't check for that. + + diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/utils/fusion_modules.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/utils/fusion_modules.py new file mode 100644 index 00000000..38ecd264 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/utils/fusion_modules.py @@ -0,0 +1,322 @@ +import torch.nn as nn +import torch +import math + + +def precompute_freqs_cis(dim: int, end: int, constant: float = 10000.0): + ''' + 计算cos和sin的值,cos值在实部,sin值在虚部,类似于 cosx+j*sinx + :param dim: q,k,v的最后一维,一般为emb_dim/head_num + :param end: 句长length + :param constant: 这里指10000 + :return: + 复数计算 torch.polar(a, t)输出, a*(cos(t)+j*sin(t)) + ''' + # freqs: 计算 1/(10000^(2i/d) ),将结果作为参数theta + # 形式化为 [theta_0, theta_1, ..., theta_(d/2-1)] + freqs = 1.0 / (constant ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim)) # [d/2] + + # 计算m + t = torch.arange(end, device=freqs.device) # [length] + # 计算m*theta + freqs = torch.outer(t, freqs).float() # [length, d/2] + # freqs形式化为 [m*theta_0, m*theta_1, ..., m*theta_(d/2-1)],其中 m=0,1,...,length-1 + + # 计算cos(m*theta)+j*sin(m*theta) + freqs_cis = torch.polar(torch.ones_like(freqs), freqs) # complex64 + # freqs_cis: [cos(m*theta_0)+j*sin(m*theta_0), cos(m*theta_1)+j*sin(m*theta_1),), ..., cos(m*theta_(d/2-1))+j*sin(m*theta_(d/2-1))] + # 其中j为虚数单位, m=0,1,...,length-1 + return freqs_cis # [length, d/2] + + +def reshape_for_broadcast(freqs_cis: torch.Tensor, x: torch.Tensor): + ndim = x.ndim + assert 0 <= 1 < ndim + assert freqs_cis.shape == (x.shape[1], x.shape[-1]) + shape = [d if i == 1 or i == ndim - 1 else 1 for i, d in enumerate(x.shape)] # (1, length, 1, d/2) + return freqs_cis.view(*shape) # [1, length, 1, d/2] + + +def apply_rotary_emb(xq: torch.Tensor, xk: torch.Tensor, q_freqs_cis: torch.Tensor,k_freqs_cis: torch.Tensor ): + # 先将xq维度变为[bs, length, head, d/2, 2], 利用torch.view_as_complex转变为复数 + # xq:[q0, q1, .., q(d-1)] 转变为 xq_: [q0+j*q1, q2+j*q3, ..., q(d-2)+j*q(d-1)] + xq_ = torch.view_as_complex(xq.float().reshape(*xq.shape[:-1], -1, 2)) # [bs, length, head, d/2] + # 同样的,xk_:[k0+j*k1, k2+j*k3, ..., k(d-2)+j*k(d-1)] + xk_ = torch.view_as_complex(xk.float().reshape(*xk.shape[:-1], -1, 2)) + + q_freqs_cis = reshape_for_broadcast(q_freqs_cis, xq_) # [1, length, 1, d/2] + k_freqs_cis = reshape_for_broadcast(k_freqs_cis, xk_) # [1, length, 1, d/2] + + # 下式xq_ * freqs_cis形式化输出,以第一个为例, 如下 + # (q0+j*q1)(cos(m*theta_0)+j*sin(m*theta_0)) = q0*cos(m*theta_0)-q1*sin(m*theta_0) + j*(q1*cos(m*theta_0)+q0*sin(m*theta_0)) + # 上式的实部为q0*cos(m*theta_0)-q1*sin(m*theta_0),虚部为q1*cos(m*theta_0)+q0*sin(m*theta_0) + # 然后通过torch.view_as_real函数,取出实部和虚部,维度由[bs, length, head, d/2]变为[bs, length, head, d/2, 2],最后一维放实部与虚部 + # 最后经flatten函数将维度拉平,即[bs, length, head, d] + # 此时xq_out形式化为 [实部0,虚部0,实部1,虚部1,..., 实部(d/2-1), 虚部(d/2-1)] + xq_out = torch.view_as_real(xq_ * q_freqs_cis).flatten(3) # [bs, length, head, d] + # 即为新生成的q + + xk_out = torch.view_as_real(xk_ * k_freqs_cis).flatten(3) + return xq_out.type_as(xq), xk_out.type_as(xk) + + +class BertSelfAttention(nn.Module): + def __init__(self, config, is_cross_attention): + super().__init__() + self.config = config + if config.hidden_size % config.num_attention_heads != 0 and not hasattr( + config, "embedding_size" + ): + raise ValueError( + "The hidden size (%d) is not a multiple of the number of attention " + "heads (%d)" % (config.hidden_size, config.num_attention_heads) + ) + + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + if is_cross_attention: + self.key = nn.Linear(config.encoder_width, self.all_head_size) + self.value = nn.Linear(config.encoder_width, self.all_head_size) + else: + self.key = nn.Linear(config.hidden_size, self.all_head_size) + self.value = nn.Linear(config.hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + self.position_embedding_type = getattr( + config, "position_embedding_type", "absolute" + ) + if ( + self.position_embedding_type == "relative_key" + or self.position_embedding_type == "relative_key_query" + ): + self.max_position_embeddings = config.max_position_embeddings + self.distance_embedding = nn.Embedding( + 2 * config.max_position_embeddings - 1, self.attention_head_size + ) + self.save_attention = False + + def save_attn_gradients(self, attn_gradients): + self.attn_gradients = attn_gradients + + def get_attn_gradients(self): + return self.attn_gradients + + def save_attention_map(self, attention_map): + self.attention_map = attention_map + + def get_attention_map(self): + return self.attention_map + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + ( + self.num_attention_heads, + self.attention_head_size, + ) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + + # If this is instantiated as a cross-attention module, the keys + # and values come from an encoder; the attention mask needs to be + # such that the encoder's padding tokens are not attended to. + is_cross_attention = encoder_hidden_states is not None + + if is_cross_attention: + key_layer = self.transpose_for_scores(self.key(encoder_hidden_states)) + value_layer = self.transpose_for_scores(self.value(encoder_hidden_states)) + attention_mask = encoder_attention_mask + elif past_key_value is not None: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + key_layer = torch.cat([past_key_value[0], key_layer], dim=2) + value_layer = torch.cat([past_key_value[1], value_layer], dim=2) + else: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + + mixed_query_layer = self.query(hidden_states) + + query_layer = self.transpose_for_scores(mixed_query_layer) + + q_freqs_cis = precompute_freqs_cis(dim=query_layer.shape[-1], end=query_layer.shape[-2], constant=10000.0).to(device=key_layer.device) + k_freqs_cis = precompute_freqs_cis(dim=key_layer.shape[-1], end=key_layer.shape[-2], constant=10000.0).to(device=key_layer.device) + + query_layer, key_layer = apply_rotary_emb(xq=query_layer.permute(0,2,1,3), xk=key_layer.permute(0,2,1,3), q_freqs_cis=q_freqs_cis, k_freqs_cis=k_freqs_cis) + query_layer = query_layer.permute(0, 2, 1, 3) + key_layer = key_layer.permute(0, 2, 1, 3) + past_key_value = (key_layer, value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + + if ( + self.position_embedding_type == "relative_key" + or self.position_embedding_type == "relative_key_query" + ): + seq_length = hidden_states.size()[1] + position_ids_l = torch.arange( + seq_length, dtype=torch.long, device=hidden_states.device + ).view(-1, 1) + position_ids_r = torch.arange( + seq_length, dtype=torch.long, device=hidden_states.device + ).view(1, -1) + distance = position_ids_l - position_ids_r + positional_embedding = self.distance_embedding( + distance + self.max_position_embeddings - 1 + ) + positional_embedding = positional_embedding.to( + dtype=query_layer.dtype + ) # fp16 compatibility + + if self.position_embedding_type == "relative_key": + relative_position_scores = torch.einsum( + "bhld,lrd->bhlr", query_layer, positional_embedding + ) + attention_scores = attention_scores + relative_position_scores + elif self.position_embedding_type == "relative_key_query": + relative_position_scores_query = torch.einsum( + "bhld,lrd->bhlr", query_layer, positional_embedding + ) + relative_position_scores_key = torch.einsum( + "bhrd,lrd->bhlr", key_layer, positional_embedding + ) + attention_scores = ( + attention_scores + + relative_position_scores_query + + relative_position_scores_key + ) + + attention_scores = attention_scores / math.sqrt(self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_mask = attention_mask.unsqueeze(1).expand_as(attention_scores) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + if is_cross_attention and self.save_attention: + self.save_attention_map(attention_probs) + attention_probs.register_hook(self.save_attn_gradients) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs_dropped = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs_dropped = attention_probs_dropped * head_mask + + context_layer = torch.matmul(attention_probs_dropped, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) + context_layer = context_layer.view(*new_context_layer_shape) + + outputs = ( + (context_layer, attention_probs) if output_attentions else (context_layer,) + ) + + outputs = outputs + (past_key_value,) + return outputs + + +class BertSelfOutput(nn.Module): + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class BertAttention(nn.Module): + def __init__(self, config, is_cross_attention=True): + super().__init__() + self.self = BertSelfAttention(config, is_cross_attention) + self.output = BertSelfOutput(config) + self.pruned_heads = set() + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + self_outputs = self.self( + hidden_states, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + ) + attention_output = self.output(self_outputs[0], hidden_states) + + outputs = (attention_output,) + self_outputs[ + 1: + ] # add attentions if we output them + return outputs + + +class ActionProjector(nn.Module): + def __init__(self, in_dim, out_dim=1024): + super(ActionProjector, self).__init__() + self.global_1d_pool = nn.AdaptiveAvgPool1d(1) + self.mlps = nn.ModuleList([ + # nn.LayerNorm(in_dim), + nn.Linear(in_dim, in_dim), + nn.GELU(), + nn.Linear(in_dim, out_dim), + nn.Dropout(0.0), + ] + ) + + def forward(self, x): + x = self.global_1d_pool(x.permute(1, 0)).permute(1, 0) + for mlp in self.mlps: + x = mlp(x) + return x + + +class FiLM(nn.Module): + def __init__(self, feature_dim, condition_dim): + super(FiLM, self).__init__() + self.scale_fc = nn.Linear(condition_dim, feature_dim) + self.shift_fc = nn.Linear(condition_dim, feature_dim) + + nn.init.zeros_(self.scale_fc.weight) + nn.init.zeros_(self.scale_fc.bias) + nn.init.zeros_(self.shift_fc.weight) + nn.init.zeros_(self.shift_fc.bias) + + def forward(self, x, condition): + # 计算缩放和偏移参数 + scale = self.scale_fc(condition) + shift = self.shift_fc(condition) + + # 应用 FiLM 调制 + return x * (1 + scale) + shift diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/utils/image_processing_qwen2_vla.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/utils/image_processing_qwen2_vla.py new file mode 100644 index 00000000..db33f7c6 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/utils/image_processing_qwen2_vla.py @@ -0,0 +1,462 @@ +# coding=utf-8 +# Copyright 2024 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Image processor class for Qwen2-VL.""" + +import math +from typing import Dict, List, Optional, Union + +import numpy as np + +from transformers.image_processing_utils import BaseImageProcessor, BatchFeature +from transformers.image_transforms import ( + convert_to_rgb, + resize, + to_channel_dimension_format, +) +from transformers.image_utils import ( + OPENAI_CLIP_MEAN, + OPENAI_CLIP_STD, + ChannelDimension, + ImageInput, + PILImageResampling, + VideoInput, + get_image_size, + infer_channel_dimension_format, + is_scaled_image, + is_valid_image, + make_list_of_images, + to_numpy_array, + valid_images, + validate_preprocess_arguments, +) +from transformers.utils import TensorType, is_vision_available, logging + + +logger = logging.get_logger(__name__) + + +if is_vision_available(): + from PIL import Image + + +def make_batched_images(images) -> List[List[ImageInput]]: + """ + Accepts images in list or nested list format, and makes a list of images for preprocessing. + + Args: + images (`Union[List[List[ImageInput]], List[ImageInput], ImageInput]`): + The input image. + + Returns: + list: A list of images. + """ + if isinstance(images, (list, tuple)) and isinstance(images[0], (list, tuple)) and is_valid_image(images[0][0]): + return [img for img_list in images for img in img_list] + + elif isinstance(images, (list, tuple)) and is_valid_image(images[0]): + return images + + elif is_valid_image(images): + return [images] + + raise ValueError(f"Could not make batched images from {images}") + + +# Copied from transformers.models.llava_next_video.image_processing_llava_next_video.make_batched_videos +def make_batched_videos(videos) -> List[VideoInput]: + if isinstance(videos, (list, tuple)) and isinstance(videos[0], (list, tuple)) and is_valid_image(videos[0][0]): + return videos + + elif isinstance(videos, (list, tuple)) and is_valid_image(videos[0]): + if isinstance(videos[0], Image.Image): + return [videos] + elif len(videos[0].shape) == 4: + return [list(video) for video in videos] + + elif is_valid_image(videos) and len(videos.shape) == 4: + return [list(videos)] + + raise ValueError(f"Could not make batched video from {videos}") + + +def smart_resize( + height: int, width: int, factor: int = 28, min_pixels: int = 56 * 56, max_pixels: int = 14 * 14 * 4 * 1280 +): + """Rescales the image so that the following conditions are met: + + 1. Both dimensions (height and width) are divisible by 'factor'. + + 2. The total number of pixels is within the range ['min_pixels', 'max_pixels']. + + 3. The aspect ratio of the image is maintained as closely as possible. + + """ + if height < factor or width < factor: + raise ValueError(f"height:{height} or width:{width} must be larger than factor:{factor}") + elif max(height, width) / min(height, width) > 200: + raise ValueError( + f"absolute aspect ratio must be smaller than 200, got {max(height, width) / min(height, width)}" + ) + h_bar = round(height / factor) * factor + w_bar = round(width / factor) * factor + if h_bar * w_bar > max_pixels: + beta = math.sqrt((height * width) / max_pixels) + h_bar = math.floor(height / beta / factor) * factor + w_bar = math.floor(width / beta / factor) * factor + elif h_bar * w_bar < min_pixels: + beta = math.sqrt(min_pixels / (height * width)) + h_bar = math.ceil(height * beta / factor) * factor + w_bar = math.ceil(width * beta / factor) * factor + return h_bar, w_bar + + +class Qwen2VLImageProcessor(BaseImageProcessor): + r""" + Constructs a Qwen2-VL image processor that dynamically resizes images based on the original images. + + Args: + do_resize (`bool`, *optional*, defaults to `True`): + Whether to resize the image's (height, width) dimensions. + resample (`PILImageResampling`, *optional*, defaults to `Resampling.BICUBIC`): + Resampling filter to use when resizing the image. + do_rescale (`bool`, *optional*, defaults to `True`): + Whether to rescale the image by the specified scale `rescale_factor`. + rescale_factor (`int` or `float`, *optional*, defaults to `1/255`): + Scale factor to use if rescaling the image. + do_normalize (`bool`, *optional*, defaults to `True`): + Whether to normalize the image. + image_mean (`float` or `List[float]`, *optional*, defaults to `[0.48145466, 0.4578275, 0.40821073]`): + Mean to use if normalizing the image. This is a float or list of floats for each channel in the image. + image_std (`float` or `List[float]`, *optional*, defaults to `[0.26862954, 0.26130258, 0.27577711]`): + Standard deviation to use if normalizing the image. This is a float or list of floats for each channel in the image. + do_convert_rgb (`bool`, *optional*, defaults to `True`): + Whether to convert the image to RGB. + min_pixels (`int`, *optional*, defaults to `56 * 56`): + The min pixels of the image to resize the image. + max_pixels (`int`, *optional*, defaults to `28 * 28 * 1280`): + The max pixels of the image to resize the image. + patch_size (`int`, *optional*, defaults to 14): + The spacial patch size of the vision encoder. + temporal_patch_size (`int`, *optional*, defaults to 2): + The temporal patch size of the vision encoder. + merge_size (`int`, *optional*, defaults to 2): + The merge size of the vision encoder to llm encoder. + """ + + model_input_names = ["pixel_values", "image_grid_thw", "pixel_values_videos", "video_grid_thw"] + + def __init__( + self, + do_resize: bool = True, + resample: PILImageResampling = PILImageResampling.BICUBIC, + do_rescale: bool = True, + rescale_factor: Union[int, float] = 1 / 255, + do_normalize: bool = True, + image_mean: Optional[Union[float, List[float]]] = None, + image_std: Optional[Union[float, List[float]]] = None, + do_convert_rgb: bool = True, + min_pixels: int = 56 * 56, + max_pixels: int = 28 * 28 * 1280, + patch_size: int = 14, + temporal_patch_size: int = 2, + merge_size: int = 2, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.do_resize = do_resize + self.resample = resample + self.do_rescale = do_rescale + self.rescale_factor = rescale_factor + self.do_normalize = do_normalize + self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN + self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD + self.min_pixels = min_pixels + self.max_pixels = max_pixels + self.patch_size = patch_size + self.temporal_patch_size = temporal_patch_size + self.merge_size = merge_size + self.size = {"min_pixels": min_pixels, "max_pixels": max_pixels} + self.do_convert_rgb = do_convert_rgb + + def _preprocess( + self, + images: Union[ImageInput, VideoInput], + do_resize: bool = None, + resample: PILImageResampling = None, + do_rescale: bool = None, + rescale_factor: float = None, + do_normalize: bool = None, + image_mean: Optional[Union[float, List[float]]] = None, + image_std: Optional[Union[float, List[float]]] = None, + do_convert_rgb: bool = None, + data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + ): + """ + Preprocess an image or batch of images. Copy of the `preprocess` method from `CLIPImageProcessor`. + + Args: + images (`ImageInput`): + Image or batch of images to preprocess. Expects pixel values ranging from 0 to 255. If pixel values range from 0 to 1, set `do_rescale=False`. + vision_info (`List[Dict]`, *optional*): + Optional list of dictionaries containing additional information about vision inputs. + do_resize (`bool`, *optional*, defaults to `self.do_resize`): + Whether to resize the image. + resample (`PILImageResampling`, *optional*, defaults to `self.resample`): + Resampling filter to use if resizing the image. This can be one of the `PILImageResampling` enums. + do_rescale (`bool`, *optional*, defaults to `self.do_rescale`): + Whether to rescale the image. + rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`): + Scale factor to use if rescaling the image. + do_normalize (`bool`, *optional*, defaults to `self.do_normalize`): + Whether to normalize the image. + image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`): + Mean to use if normalizing the image. Can be a float or a list of floats corresponding to the number of channels in the image. + image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`): + Standard deviation to use if normalizing the image. Can be a float or a list of floats corresponding to the number of channels in the image. + do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`): + Whether to convert the image to RGB. + data_format (`ChannelDimension`, *optional*, defaults to `ChannelDimension.FIRST`): + The channel dimension format for the output image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - Unset: Use the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + """ + images = make_list_of_images(images) + + if do_convert_rgb: + images = [convert_to_rgb(image) for image in images] + + # All transformations expect numpy arrays. + images = [to_numpy_array(image) for image in images] + + if is_scaled_image(images[0]) and do_rescale: + logger.warning_once( + "It looks like you are trying to rescale already rescaled images. If the input" + " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again." + ) + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0]) + + height, width = get_image_size(images[0], channel_dim=input_data_format) + resized_height, resized_width = height, width + processed_images = [] + for image in images: + if do_resize: + resized_height, resized_width = smart_resize( + height, + width, + factor=self.patch_size * self.merge_size, + min_pixels=self.min_pixels, + max_pixels=self.max_pixels, + ) + image = resize( + image, size=(resized_height, resized_width), resample=resample, input_data_format=input_data_format + ) + + if do_rescale: + image = self.rescale(image, scale=rescale_factor, input_data_format=input_data_format) + + if do_normalize: + image = self.normalize( + image=image, mean=image_mean, std=image_std, input_data_format=input_data_format + ) + + image = to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) + processed_images.append(image) + + patches = np.array(processed_images) + if data_format == ChannelDimension.LAST: + patches = patches.transpose(0, 3, 1, 2) + if patches.shape[0] == 1: + patches = np.tile(patches, (self.temporal_patch_size, 1, 1, 1)) + channel = patches.shape[1] + grid_t = patches.shape[0] // self.temporal_patch_size + grid_h, grid_w = resized_height // self.patch_size, resized_width // self.patch_size + patches = patches.reshape( + grid_t, + self.temporal_patch_size, + channel, + grid_h // self.merge_size, + self.merge_size, + self.patch_size, + grid_w // self.merge_size, + self.merge_size, + self.patch_size, + ) + patches = patches.transpose(0, 3, 6, 4, 7, 2, 1, 5, 8) + flatten_patches = patches.reshape( + grid_t * grid_h * grid_w, channel * self.temporal_patch_size * self.patch_size * self.patch_size + ) + + return flatten_patches, (grid_t, grid_h, grid_w) + + def preprocess( + self, + images: ImageInput, + videos: VideoInput = None, + do_resize: bool = None, + size: Dict[str, int] = None, + resample: PILImageResampling = None, + do_rescale: bool = None, + rescale_factor: float = None, + do_normalize: bool = None, + image_mean: Optional[Union[float, List[float]]] = None, + image_std: Optional[Union[float, List[float]]] = None, + do_convert_rgb: bool = None, + return_tensors: Optional[Union[str, TensorType]] = None, + data_format: Optional[ChannelDimension] = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + ): + """ + Args: + images (`ImageInput`): + Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If + passing in images with pixel values between 0 and 1, set `do_rescale=False`. + videos (`VideoInput`): + Video to preprocess. Expects a single or batch of videos with pixel values ranging from 0 to 255. If + passing in videos with pixel values between 0 and 1, set `do_rescale=False`. + do_resize (`bool`, *optional*, defaults to `self.do_resize`): + Whether to resize the image. + size (`Dict[str, int]`, *optional*, defaults to `self.size`): + Size of the image after resizing. Shortest edge of the image is resized to size["shortest_edge"], with + the longest edge resized to keep the input aspect ratio. + resample (`int`, *optional*, defaults to `self.resample`): + Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only + has an effect if `do_resize` is set to `True`. + do_rescale (`bool`, *optional*, defaults to `self.do_rescale`): + Whether to rescale the image. + rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`): + Rescale factor to rescale the image by if `do_rescale` is set to `True`. + do_normalize (`bool`, *optional*, defaults to `self.do_normalize`): + Whether to normalize the image. + image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`): + Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`. + image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`): + Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to + `True`. + do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`): + Whether to convert the image to RGB. + return_tensors (`str` or `TensorType`, *optional*): + The type of tensors to return. Can be one of: + - Unset: Return a list of `np.ndarray`. + - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`. + - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`. + - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`. + - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. + data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): + The channel dimension format for the output image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - Unset: Use the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + + """ + do_resize = do_resize if do_resize is not None else self.do_resize + size = size if size is not None else self.size + resample = resample if resample is not None else self.resample + do_rescale = do_rescale if do_rescale is not None else self.do_rescale + rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor + do_normalize = do_normalize if do_normalize is not None else self.do_normalize + image_mean = image_mean if image_mean is not None else self.image_mean + image_std = image_std if image_std is not None else self.image_std + do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb + + if images is not None: + images = make_batched_images(images) + if videos is not None: + videos = make_batched_videos(videos) + + if images is not None and not valid_images(images): + raise ValueError( + "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, " + "torch.Tensor, tf.Tensor or jax.ndarray." + ) + + validate_preprocess_arguments( + rescale_factor=rescale_factor, + do_normalize=do_normalize, + image_mean=image_mean, + image_std=image_std, + do_resize=do_resize, + size=size, + resample=resample, + ) + + if images is not None: + pixel_values, vision_grid_thws = [], [] + for image in images: + patches, image_grid_thw = self._preprocess( + image, + do_resize=do_resize, + resample=resample, + do_rescale=do_rescale, + rescale_factor=rescale_factor, + do_normalize=do_normalize, + image_mean=image_mean, + image_std=image_std, + data_format=data_format, + do_convert_rgb=do_convert_rgb, + input_data_format=input_data_format, + ) + pixel_values.extend(patches) + vision_grid_thws.append(image_grid_thw) + pixel_values = np.array(pixel_values) + vision_grid_thws = np.array(vision_grid_thws) + data = {"pixel_values": pixel_values, "image_grid_thw": vision_grid_thws} + + if videos is not None: + pixel_values, vision_grid_thws = [], [] + for images in videos: + patches, video_grid_thw = self._preprocess( + images, + do_resize=do_resize, + resample=resample, + do_rescale=do_rescale, + rescale_factor=rescale_factor, + do_normalize=do_normalize, + image_mean=image_mean, + image_std=image_std, + data_format=data_format, + do_convert_rgb=do_convert_rgb, + input_data_format=input_data_format, + ) + pixel_values.extend(patches) + vision_grid_thws.append(video_grid_thw) + pixel_values = np.array(pixel_values) + vision_grid_thws = np.array(vision_grid_thws) + data = {"pixel_values_videos": pixel_values, "video_grid_thw": vision_grid_thws} + + return BatchFeature(data=data, tensor_type=return_tensors) + +from transformers import AutoProcessor +AutoProcessor.register("Qwen2VLImageProcessor", Qwen2VLImageProcessor) + diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/utils/processing_qwen2_vla.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/utils/processing_qwen2_vla.py new file mode 100644 index 00000000..25c30273 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/utils/processing_qwen2_vla.py @@ -0,0 +1,178 @@ +# coding=utf-8 +# Copyright 2024 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Processor class for Qwen2-VL. +""" + +from typing import List, Union + +from transformers.feature_extraction_utils import BatchFeature +from transformers.image_utils import ImageInput, VideoInput +from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack +from transformers.tokenization_utils_base import PreTokenizedInput, TextInput +from transformers.utils import logging + + +logger = logging.get_logger(__name__) + + +class Qwen2VLProcessorKwargs(ProcessingKwargs, total=False): + _defaults = { + "text_kwargs": { + "padding": False, + }, + } + + +class Qwen2VLProcessor(ProcessorMixin): + r""" + Constructs a Qwen2-VL processor which wraps a Qwen2-VL image processor and a Qwen2 tokenizer into a single processor. + [`Qwen2VLProcessor`] offers all the functionalities of [`Qwen2VLImageProcessor`] and [`Qwen2TokenizerFast`]. See the + [`~Qwen2VLProcessor.__call__`] and [`~Qwen2VLProcessor.decode`] for more information. + Args: + image_processor ([`Qwen2VLImageProcessor`], *optional*): + The image processor is a required input. + tokenizer ([`Qwen2TokenizerFast`], *optional*): + The tokenizer is a required input. + chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages + in a chat into a tokenizable string. + """ + + attributes = ["image_processor", "tokenizer"] + valid_kwargs = ["chat_template"] + image_processor_class = "Qwen2VLImageProcessor" + tokenizer_class = ("Qwen2Tokenizer", "Qwen2TokenizerFast") + + def __init__(self, image_processor=None, tokenizer=None, chat_template=None, **kwargs): + super().__init__(image_processor, tokenizer, chat_template=chat_template) + + def __call__( + self, + images: ImageInput = None, + text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None, + videos: VideoInput = None, + **kwargs: Unpack[Qwen2VLProcessorKwargs], + ) -> BatchFeature: + """ + Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text` + and `kwargs` arguments to Qwen2TokenizerFast's [`~Qwen2TokenizerFast.__call__`] if `text` is not `None` to encode + the text. To prepare the vision inputs, this method forwards the `vision_infos` and `kwrags` arguments to + Qwen2VLImageProcessor's [`~Qwen2VLImageProcessor.__call__`] if `vision_infos` is not `None`. + + Args: + images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`, `List[torch.Tensor]`): + The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch + tensor. Both channels-first and channels-last formats are supported. + text (`str`, `List[str]`, `List[List[str]]`): + The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings + (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set + `is_split_into_words=True` (to lift the ambiguity with a batch of sequences). + videos (`np.ndarray`, `torch.Tensor`, `List[np.ndarray]`, `List[torch.Tensor]`): + The image or batch of videos to be prepared. Each video can be a 4D NumPy array or PyTorch + tensor, or a nested list of 3D frames. Both channels-first and channels-last formats are supported. + return_tensors (`str` or [`~utils.TensorType`], *optional*): + If set, will return tensors of a particular framework. Acceptable values are: + - `'tf'`: Return TensorFlow `tf.constant` objects. + - `'pt'`: Return PyTorch `torch.Tensor` objects. + - `'np'`: Return NumPy `np.ndarray` objects. + - `'jax'`: Return JAX `jnp.ndarray` objects. + + Returns: + [`BatchFeature`]: A [`BatchFeature`] with the following fields: + + - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`. + - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when + `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not + `None`). + - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`. + - **pixel_values_videos** -- Pixel values of videos to be fed to a model. Returned when `videos` is not `None`. + - **image_grid_thw** -- List of image 3D grid in LLM. Returned when `images` is not `None`. + - **video_grid_thw** -- List of video 3D grid in LLM. Returned when `videos` is not `None`. + """ + output_kwargs = self._merge_kwargs( + Qwen2VLProcessorKwargs, + tokenizer_init_kwargs=self.tokenizer.init_kwargs, + **kwargs, + ) + if images is not None: + image_inputs = self.image_processor(images=images, videos=None, **output_kwargs["images_kwargs"]) + image_grid_thw = image_inputs["image_grid_thw"] + else: + image_inputs = {} + image_grid_thw = None + + if videos is not None: + videos_inputs = self.image_processor(images=None, videos=videos, **output_kwargs["videos_kwargs"]) + video_grid_thw = videos_inputs["video_grid_thw"] + else: + videos_inputs = {} + video_grid_thw = None + + if not isinstance(text, list): + text = [text] + + if image_grid_thw is not None: + merge_length = self.image_processor.merge_size**2 + index = 0 + for i in range(len(text)): + while "<|image_pad|>" in text[i]: + text[i] = text[i].replace( + "<|image_pad|>", "<|placeholder|>" * (image_grid_thw[index].prod() // merge_length), 1 + ) + index += 1 + text[i] = text[i].replace("<|placeholder|>", "<|image_pad|>") + + if video_grid_thw is not None: + merge_length = self.image_processor.merge_size**2 + index = 0 + for i in range(len(text)): + while "<|video_pad|>" in text[i]: + text[i] = text[i].replace( + "<|video_pad|>", "<|placeholder|>" * (video_grid_thw[index].prod() // merge_length), 1 + ) + index += 1 + text[i] = text[i].replace("<|placeholder|>", "<|video_pad|>") + + text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"]) + + return BatchFeature(data={**text_inputs, **image_inputs, **videos_inputs}) + + def batch_decode(self, *args, **kwargs): + """ + This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please + refer to the docstring of this method for more information. + """ + return self.tokenizer.batch_decode(*args, **kwargs) + + def decode(self, *args, **kwargs): + """ + This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to + the docstring of this method for more information. + """ + return self.tokenizer.decode(*args, **kwargs) + + @property + def model_input_names(self): + tokenizer_input_names = self.tokenizer.model_input_names + image_processor_input_names = self.image_processor.model_input_names + return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names)) + +from transformers import AutoProcessor +AutoProcessor.register("Qwen2VLProcessor", Qwen2VLProcessor) \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/utils/robot_data_processor.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/utils/robot_data_processor.py new file mode 100644 index 00000000..d1bcc61e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/dex_vla/utils/robot_data_processor.py @@ -0,0 +1,157 @@ +from PIL import Image +import numpy as np +from torchvision.transforms.functional import to_pil_image, to_tensor +import torchvision.transforms as transforms +import torch +from qwen_vl_utils import process_vision_info +from qwen_vl_utils import * +class DexVLAProcess: + def __init__( + self, + language=None, + tokenizer=None, + max_seq_len=512, + multimodal_processor=None, + camera_names=None, + data_args=None, + ): + super().__init__() + self.tokenizer = tokenizer + self.max_seq_len = max_seq_len + self.camera_names = camera_names + # self.language = language + self.multimodal_processor = multimodal_processor + self.data_args = data_args + + def preprocess_image(self, image, size=224): + # Model has been trained to handle images of different aspects ratios + # resized to 224x224 in the range [-1, 1]. Bilinear and antialias resize + # options are helpful to improve quality in some tasks. + image = np.asarray(image) + if image.ndim == 2: # Convert image without last channel into greyscale. + image = np.stack((image,) * 3, axis=-1) + image = image[..., :3] # Remove alpha layer. + assert image.shape[-1] == 3 + + image_pil = to_pil_image(image) + + # Step 2: Define the resize transformation + resize_transform = transforms.Resize((size, size), interpolation=transforms.InterpolationMode.BILINEAR) + + # Step 3: Apply the resize transformation + image_resized_pil = resize_transform(image_pil) + + # Step 4: Convert back to tensor if needed + image_resized = to_tensor(image_resized_pil) + return image.numpy() / 127.5 - 1.0 # [0, 255]->[-1,1] + + def qwen2_image_preprocess(self, each, camera_name): + ele = { + # "resized_height": None, + # "resized_width": None + } + each = Image.fromarray(each.squeeze(0).permute(1, 2, 0).numpy().astype(np.uint8)) + ele['image'] = each + if 'wrist' in camera_name: + w, h = eval(self.data_args.image_size_wrist) + ele['resized_height'] = h + ele['resized_width'] = w + else: + ele['resized_height'] = each.height + ele['resized_width'] = each.width + each = fetch_image(ele) + return torch.from_numpy(np.array(each)) + + def forward_process(self, sample, use_reasoning=True): + if sample['image'].ndim == 5 and sample['image'].shape[1] > 2: + video = True + else: + video = False + messages = self.datastruct_droid2llava(sample, video=video) + + data_dict = dict( + messages=messages, + images=None + ) + + image_data = torch.chunk(sample['image'], sample['image'].shape[0], 0) + + images_list = [] + + for i, each in enumerate(image_data): + if each.ndim == 4: + img_pil = self.qwen2_image_preprocess(each, self.camera_names[i]) + else: + img_pil = [] + for temp in each.squeeze(0): + img_pil.append(self.qwen2_image_preprocess(temp, self.camera_names[i])) + img_pil = torch.stack(img_pil, 0) + images_list.append(img_pil) + # TODO RESIZE + # image_data = image_data / 255.0 + if video: + image_data = None + video_inputs = images_list + else: + image_data = images_list + video_inputs = None + + text = self.multimodal_processor.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True + ) + # image_inputs, video_inputs = process_vision_info(dataset) + # text = text[:-23] + model_inputs = self.multimodal_processor( + text=text, + images=image_data, + videos=video_inputs, + padding=True, + return_tensors="pt", + ) + input_labels = torch.ones_like(model_inputs['input_ids']) * -100 + if use_reasoning: + answer = sample['reasoning'] + "Next action:" + '<|im_end|>' + else: + answer = 'None.' + '<|im_end|>' + + output_text = self.tokenizer(answer, padding=True, return_tensors="pt") + output_labels = output_text['input_ids'] + model_inputs['input_ids'] = torch.cat((model_inputs['input_ids'], output_text['input_ids']), dim=-1) + model_inputs['attention_mask'] = torch.cat((model_inputs['attention_mask'], output_text['attention_mask']), dim=-1) + labels = torch.cat((input_labels, output_labels), dim=-1) + data_dict['state'] = sample['state'] + data_dict['action'] = sample['action'] + data_dict['is_pad'] = sample['is_pad'] + data_dict['labels'] = labels + data_dict['raw_images'] = sample['image'] + for k, v in model_inputs.items(): + data_dict[k] = v + return data_dict + + def datastruct_droid2llava(self, sample, video=False): + len_image = sample['image'].shape[0] + + messages = [ + { + "role": "user", + "content": [], + }, + # {"role": "assistant", "content": f''}, + ] + + for i in range(len_image): + if video: + messages[0]['content'].append({ + "type": "video", + "video": None, + }) + else: + messages[0]['content'].append({ + "type": "image", + "image": None, + }) + messages[0]['content'].append({"type": "text", "text": f""}) + messages[0]['content'][-1]['text'] = sample['raw_lang'] + # messages[1]['content'] = sample['reasoning'] + "Next action:" + # print(sample['obs']['raw_language'].decode('utf-8')) + return messages \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/eval.sh b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/eval.sh new file mode 100644 index 00000000..8f66da7a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/eval.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +policy_name=DexVLA +task_name=task_name_you_test +task_config=demo_clean +ckpt_setting=0 +seed=0 +gpu_id=0 +# [TODO] add parameters here + +export CUDA_VISIBLE_DEVICES=${gpu_id} +echo -e "\033[33mgpu id (to use): ${gpu_id}\033[0m" + +cd ../.. # move to root + +python script/eval_policy.py --config policy/$policy_name/deploy_policy.yml \ + --overrides \ + --task_name ${task_name} \ + --task_config ${task_config} \ + --ckpt_setting ${ckpt_setting} \ + --seed ${seed} \ + --policy_name ${policy_name} + # [TODO] add parameters here diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/evaluate/__init__.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/evaluate/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/evaluate/vla_policy/__init__.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/evaluate/vla_policy/__init__.py new file mode 100644 index 00000000..8b15938c --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/evaluate/vla_policy/__init__.py @@ -0,0 +1,2 @@ +from .paligemma_vla_policy import * +from .qwen2_vla_policy import * \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/evaluate/vla_policy/paligemma_vla_policy.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/evaluate/vla_policy/paligemma_vla_policy.py new file mode 100644 index 00000000..4c9fefb2 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/evaluate/vla_policy/paligemma_vla_policy.py @@ -0,0 +1,50 @@ +import torch +import cv2 +from PIL import Image +from transformers import AutoModelForMaskedLM, AutoTokenizer, AutoModel, AutoConfig, AutoModelForMaskedLM +import numpy as np +CAMERA_VIEWS=['cam_bottom', 'cam_top', 'cam_left_wrist', 'cam_right_wrist'] + +from dex_vla.model_load_utils import load_model_for_eval +class paligemma_vla_policy: + def __init__(self, policy_config, data_args=None): + super(paligemma_vla_policy).__init__() + self.load_policy(policy_config) + self.history_len = policy_config['history_image_length'] + self.data_args = data_args + + def load_policy(self, policy_config): + self.policy_config = policy_config + # self.conv = conv_templates[policy_config['conv_mode']].copy() + model_base = policy_config["model_base"] if policy_config[ + 'enable_lora'] else None + model_path = policy_config["model_path"] + + self.tokenizer, self.policy, self.multimodal_processor, self.context_len = load_model_for_eval(model_path=model_path, + model_base=model_base, policy_config=policy_config) + # self.tokenizer.add_special_tokens({'additional_special_tokens': ["[SOA]"]}) + + self.config = AutoConfig.from_pretrained('/'.join(model_path.split('/')[:-1]), trust_remote_code=True) + + def process_batch_to_qwen2_vla(self, curr_image, robo_state, raw_lang): + curr_image = curr_image[-self.history_len:] + if len(curr_image) == 1 and self.history_len > 1: + curr_image.append(curr_image[0]) + curr_image = torch.cat(curr_image, dim=0).permute((1,0,2,3,4)) # 4,2,3,240,320 the second dim is temporal + else: + # if len(curr_image.shape) == 5: # 1,2,3,270,480 + curr_image = curr_image[-1].squeeze(0) + + # image_data = torch.chunk(curr_image, curr_image.shape[0], dim=0) # left, right ,wrist + # image_list = [] + # for each in image_data: + # each = cv2.resize(cv2.cvtColor(each.squeeze().permute(1,2,0).cpu().numpy(), cv2.COLOR_BGRA2BGR), (224, 224)) + # image_list.append(torch.tensor(each).permute(2,0,1)) + # image_data = torch.stack(image_list, dim=0) + curr_image = curr_image.to(torch.int64).unsqueeze(0) + model_inputs = self.multimodal_processor(text=raw_lang, images=curr_image, return_tensors="pt").to(device=self.policy.device) + model_inputs['pixel_values'] = model_inputs['pixel_values'] + data_dict = dict(states=robo_state) + for k, v in model_inputs.items(): + data_dict[k] = v + return data_dict \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/evaluate/vla_policy/qwen2_vla_policy.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/evaluate/vla_policy/qwen2_vla_policy.py new file mode 100644 index 00000000..f103c224 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/evaluate/vla_policy/qwen2_vla_policy.py @@ -0,0 +1,116 @@ +import torch + +from PIL import Image +from qwen_vl_utils import fetch_image +from transformers import AutoModelForMaskedLM, AutoTokenizer, AutoModel, AutoConfig, AutoModelForMaskedLM +import numpy as np +CAMERA_VIEWS=['cam_bottom', 'cam_top', 'cam_left_wrist', 'cam_right_wrist'] + +from dex_vla.model_load_utils import load_model_for_eval +class qwen2_vla_policy: + def __init__(self, policy_config, data_args=None): + super(qwen2_vla_policy).__init__() + self.load_policy(policy_config) + self.history_len = policy_config['history_image_length'] + self.data_args = data_args + + def load_policy(self, policy_config): + self.policy_config = policy_config + # self.conv = conv_templates[policy_config['conv_mode']].copy() + model_base = policy_config["model_base"] if policy_config[ + 'enable_lora'] else None + model_path = policy_config["model_path"] + + self.tokenizer, self.policy, self.multimodal_processor, self.context_len = load_model_for_eval(model_path=model_path, + model_base=model_base, policy_config=policy_config) + # self.tokenizer.add_special_tokens({'additional_special_tokens': ["[SOA]"]}) + + paths = model_path.split('/')[:-1] + if 'checkpoint' in paths[-1]: + paths = paths[:-1] + self.config = AutoConfig.from_pretrained(model_path, trust_remote_code=True) + def datastruct_droid2qwen2vla(self, raw_lang, image_len): + messages = [ + { + "role": "user", + "content": [], + }, + # {"role": "assistant", "content": f''}, + ] + + for i in range(image_len): + messages[0]['content'].append({ + "type": "image", + "image": None, + }) + + messages[0]['content'].append({"type": "text", "text": f""}) + + messages[0]['content'][-1]['text'] = raw_lang + # messages[1]['content'] = sample['reasoning'] + "Next action:" + # print(sample['obs']['raw_language'].decode('utf-8')) + return messages + + def qwen2_image_preprocess(self, each, camera_name): + ele = { + # "resized_height": None, + # "resized_width": None + } + each = Image.fromarray(each.squeeze(0).permute(1, 2, 0).cpu().numpy().astype(np.uint8)) + ele['image'] = each + # if 'wrist' in camera_name: + # # w, h = eval(self.data_args.image_size_wrist) + # w,h=224,224 + # ele['resized_height'] = h + # ele['resized_width'] = w + # else: + # ele['resized_height'] = each.height + # ele['resized_width'] = each.width + ele['resized_height'] = each.height + ele['resized_width'] = each.width + each = fetch_image(ele) + return torch.from_numpy(np.array(each)) + + def process_batch_to_qwen2_vla(self, curr_image, robo_state, raw_lang): + curr_image = curr_image[-self.history_len:] + if len(curr_image) == 1 and self.history_len > 1: + curr_image.append(curr_image[0]) + curr_image = torch.cat(curr_image, dim=0).permute((1,0,2,3,4)) # 4,2,3,240,320 the second dim is temporal + else: + # if len(curr_image.shape) == 5: # 1,2,3,270,480 + curr_image = curr_image[-1].squeeze(0) + + messages = self.datastruct_droid2qwen2vla(raw_lang, curr_image.shape[0]) + image_data = torch.chunk(curr_image, curr_image.shape[0], dim=0) # left, right ,wrist + image_list = [] + for i, each in enumerate(image_data[:]): + each = each.squeeze(0) + if each.ndim == 3: + img_pil = self.qwen2_image_preprocess(each, CAMERA_VIEWS[i]) + else: + img_pil = [] + for temp in each.squeeze(0): + img_pil.append(self.qwen2_image_preprocess(temp, CAMERA_VIEWS[i])) + img_pil = torch.stack(img_pil, 0) + image_list.append(img_pil) + + # TODO RESIZE + # image_data = image_data / 255.0 + image_data = image_list + text = self.multimodal_processor.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True + ) + # image_inputs, video_inputs = process_vision_info(dataset) + # text = text[:-23] + video_inputs = None + model_inputs = self.multimodal_processor( + text=text, + images=image_data, + videos=video_inputs, + padding=True, + return_tensors="pt", + ) + data_dict = dict(states=robo_state) + for k, v in model_inputs.items(): + data_dict[k] = v + return data_dict \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/main.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/main.py new file mode 100644 index 00000000..4efa0ae7 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/main.py @@ -0,0 +1,90 @@ +import safetensors +import os +import torch +from safetensors import safe_open + + +path = '/home/rl/Downloads/output/checkpoint-4' +path = '/media/rl/HDD/data/multi_head_train_results/aloha_qwen2_vla/qwen2_vl_2B/qwen2_vl_only_folding_shirt_lora_ema_finetune_dit_h_4w_steps/checkpoint-30000' +def compare_lora_weights(): + ckpt = safe_open(os.path.join(path, 'adapter_model.safetensors'), framework='pt') + ema_ckpt = safe_open(os.path.join(path, 'ema', 'adapter_model.safetensors'), framework='pt') + + for k in ckpt.keys(): + # print(f">>>>>>>>>>>>>>>>>>>>>>{k}<<<<<<<<<<<<<<<<<<<<<<<") + print(k, torch.equal(ckpt.get_tensor(k),ema_ckpt.get_tensor(k))) + + pass + +def compare_non_lora_weights(): + ckpt = torch.load(os.path.join(path, 'non_lora_trainables.bin')) + try: + ema_ckpt = torch.load(os.path.join(path, 'ema_non_lora_trainables.bin')) + except Exception as e: + print(e) + ema_ckpt = torch.load(os.path.join(path, 'ema', 'non_lora_trainables.bin')) + + for k in ckpt.keys(): + # print(f">>>>>>>>>>>>>>>>>>>>>>{k}<<<<<<<<<<<<<<<<<<<<<<<") + print(k, torch.equal(ckpt[k], ema_ckpt[k])) + + pass + +def compare_zero_weights(tag='global_step30000'): + ckpt = torch.load(os.path.join(path, tag, 'bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt'), map_location=torch.device('cpu'))['optimizer_state_dict'] + ema_ckpt = torch.load(os.path.join(path, 'ema', tag, 'bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt'), map_location=torch.device('cpu'))['optimizer_state_dict'] + print(ckpt.keys()) + for k in ckpt.keys(): + # print(f">>>>>>>>>>>>>>>>>>>>>>{k}<<<<<<<<<<<<<<<<<<<<<<<") + print(k, torch.equal(ckpt[k], ema_ckpt[k])) + + pass + +def compare_ema_weights(): + ckpt = torch.load(os.path.join(path, 'non_lora_trainables.bin'), map_location=torch.device('cpu')) + ema_ckpt = torch.load(os.path.join(path, 'ema_weights_trainable.pth'), map_location=torch.device('cpu')) + # print(len(ema_ckpt.keys()), len(ckpt.keys())) + for k in ema_ckpt.keys(): + # print(f">>>>>>>>>>>>>>>>>>>>>>{k}<<<<<<<<<<<<<<<<<<<<<<<") + if 'policy_head' in k: + bool_matrix = ckpt[k] == ema_ckpt[k] + false_indices = torch.where(bool_matrix == False) + print(k, bool_matrix, false_indices) + for i,j in zip(false_indices[0], false_indices[1]): + print(ckpt[k].shape, ckpt[k][i][j].to(ema_ckpt[k].dtype).item(), ema_ckpt[k][i][j].item()) + break + if k in ckpt.keys(): + print(k, ckpt[k].dtype, ema_ckpt[k].dtype, torch.equal(ckpt[k].to(ema_ckpt[k].dtype), ema_ckpt[k])) + else: + print(f'no weights for {k} in ckpt') + + pass +def debug(): + state_dict = model.state_dict() + ema_state_dict = self.ema.averaged_model.state_dict() + for k in ema_state_dict.keys(): + print(k, state_dict[k].requires_grad, torch.equal(state_dict[k], ema_state_dict[k])) + + + +def check_norm_stats(): + path = '/media/rl/HDD/data/multi_head_train_results/aloha_qwen2_vla/qwen2_vl_2B/qwen2_vl_calculate_norm_stats/dataset_stats.pkl' + import pickle + + with open(path, 'rb') as f: + stats = pickle.load(f) + gripper = {} + for k, v in stats.items(): + gripper[k] = {} + for kk, vv in v.items(): + gripper[k][kk] = [vv[6], vv[13]] + pass + +if __name__ == '__main__': + # compare_non_lora_weights() + # compare_zero_weights() + # compare_ema_weights() + # ema_ckpt = torch.load(os.path.join("/home/rl/Downloads/output/checkpoint-2", 'ema_weights.pth'), map_location=torch.device('cpu')) + # for k,v in ema_ckpt.items(): + # if + check_norm_stats() diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/LICENSE b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/LICENSE new file mode 100644 index 00000000..b1395e94 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2020 - present, Facebook, Inc + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/README.md b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/README.md new file mode 100644 index 00000000..500b1b8d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/README.md @@ -0,0 +1,9 @@ +This part of the codebase is modified from DETR https://github.com/facebookresearch/detr under APACHE 2.0. + + @article{Carion2020EndtoEndOD, + title={End-to-End Object Detection with Transformers}, + author={Nicolas Carion and Francisco Massa and Gabriel Synnaeve and Nicolas Usunier and Alexander Kirillov and Sergey Zagoruyko}, + journal={ArXiv}, + year={2020}, + volume={abs/2005.12872} + } \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/__init__.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/__init__.py new file mode 100644 index 00000000..940686be --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/__init__.py @@ -0,0 +1,2 @@ +from .models.transformer_diffusion.modeling_dit_diffusion import * +from .models.transformer_diffusion.configuration_dit_diffusion import * diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/main.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/main.py new file mode 100644 index 00000000..eb6b77fa --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/main.py @@ -0,0 +1,130 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import argparse +from pathlib import Path + +import numpy as np +import torch +from .models import build_ACT_model, build_CNNMLP_model + +import IPython +e = IPython.embed + +def get_args_parser(): + parser = argparse.ArgumentParser('Set transformer detector', add_help=False) + parser.add_argument('--lr', default=1e-4, type=float) # will be overridden + parser.add_argument('--lr_backbone', default=1e-5, type=float) # will be overridden + parser.add_argument('--batch_size', default=2, type=int) # not used + parser.add_argument('--weight_decay', default=1e-4, type=float) + parser.add_argument('--epochs', default=300, type=int) # not used + parser.add_argument('--lr_drop', default=200, type=int) # not used + parser.add_argument('--clip_max_norm', default=0.1, type=float, # not used + help='gradient clipping max norm') + + # Model parameters + # * Backbone + parser.add_argument('--backbone', default='resnet18', type=str, # will be overridden + help="Name of the convolutional backbone to use") + parser.add_argument('--dilation', action='store_true', + help="If true, we replace stride with dilation in the last convolutional block (DC5)") + parser.add_argument('--position_embedding', default='sine', type=str, choices=('sine', 'learned'), + help="Type of positional embedding to use on top of the image features") + parser.add_argument('--camera_names', default=[], type=list, # will be overridden + help="A list of camera names") + + # * Transformer + parser.add_argument('--enc_layers', default=4, type=int, # will be overridden + help="Number of encoding layers in the transformer") + parser.add_argument('--dec_layers', default=6, type=int, # will be overridden + help="Number of decoding layers in the transformer") + parser.add_argument('--dim_feedforward', default=2048, type=int, # will be overridden + help="Intermediate size of the feedforward layers in the transformer blocks") + parser.add_argument('--hidden_dim', default=256, type=int, # will be overridden + help="Size of the embeddings (dimension of the transformer)") + parser.add_argument('--dropout', default=0.1, type=float, + help="Dropout applied in the transformer") + parser.add_argument('--nheads', default=8, type=int, # will be overridden + help="Number of attention heads inside the transformer's attentions") + parser.add_argument('--num_queries', default=400, type=int, # will be overridden + help="Number of query slots") + parser.add_argument('--pre_norm', action='store_true') + + # * Segmentation + parser.add_argument('--masks', action='store_true', + help="Train segmentation head if the flag is provided") + + # repeat args in imitate_episodes just to avoid error. Will not be used + parser.add_argument('--eval', action='store_true') + parser.add_argument('--onscreen_render', action='store_true') + parser.add_argument('--ckpt_dir', action='store', type=str, help='ckpt_dir', required=True) + parser.add_argument('--policy_class', action='store', type=str, help='policy_class, capitalize', required=True) + parser.add_argument('--task_name', action='store', type=str, help='task_name', required=True) + parser.add_argument('--seed', action='store', type=int, help='seed', required=True) + parser.add_argument('--num_steps', action='store', type=int, help='num_epochs', required=True) + parser.add_argument('--kl_weight', action='store', type=int, help='KL Weight', required=False) + parser.add_argument('--chunk_size', action='store', type=int, help='chunk_size', required=False) + parser.add_argument('--temporal_agg', action='store_true') + + parser.add_argument('--use_vq', action='store_true') + parser.add_argument('--vq_class', action='store', type=int, help='vq_class', required=False) + parser.add_argument('--vq_dim', action='store', type=int, help='vq_dim', required=False) + parser.add_argument('--load_pretrain', action='store_true', default=False) + parser.add_argument('--action_dim', action='store', type=int, required=False) + parser.add_argument('--eval_every', action='store', type=int, default=500, help='eval_every', required=False) + parser.add_argument('--validate_every', action='store', type=int, default=500, help='validate_every', required=False) + parser.add_argument('--save_every', action='store', type=int, default=500, help='save_every', required=False) + parser.add_argument('--resume_ckpt_path', action='store', type=str, help='load_ckpt_path', required=False) + parser.add_argument('--no_encoder', action='store_true') + parser.add_argument('--skip_mirrored_data', action='store_true') + parser.add_argument('--actuator_network_dir', action='store', type=str, help='actuator_network_dir', required=False) + parser.add_argument('--history_len', action='store', type=int) + parser.add_argument('--future_len', action='store', type=int) + parser.add_argument('--prediction_len', action='store', type=int) + + return parser + + +def build_ACT_model_and_optimizer(args_override): + parser = argparse.ArgumentParser('DETR training and evaluation script', parents=[get_args_parser()]) + args = parser.parse_args() + + for k, v in args_override.items(): + setattr(args, k, v) + + model = build_ACT_model(args) + model.cuda() + + param_dicts = [ + {"params": [p for n, p in model.named_parameters() if "backbone" not in n and p.requires_grad]}, + { + "params": [p for n, p in model.named_parameters() if "backbone" in n and p.requires_grad], + "lr": args.lr_backbone, + }, + ] + optimizer = torch.optim.AdamW(param_dicts, lr=args.lr, + weight_decay=args.weight_decay) + + return model, optimizer + + +def build_CNNMLP_model_and_optimizer(args_override): + parser = argparse.ArgumentParser('DETR training and evaluation script', parents=[get_args_parser()]) + args = parser.parse_args() + + for k, v in args_override.items(): + setattr(args, k, v) + + model = build_CNNMLP_model(args) + model.cuda() + + param_dicts = [ + {"params": [p for n, p in model.named_parameters() if "backbone" not in n and p.requires_grad]}, + { + "params": [p for n, p in model.named_parameters() if "backbone" in n and p.requires_grad], + "lr": args.lr_backbone, + }, + ] + optimizer = torch.optim.AdamW(param_dicts, lr=args.lr, + weight_decay=args.weight_decay) + + return model, optimizer + diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/models/transformer_diffusion/__pycache__/configuration_dit_diffusion.cpython-310.pyc b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/models/transformer_diffusion/__pycache__/configuration_dit_diffusion.cpython-310.pyc new file mode 100644 index 00000000..b6aa22ee Binary files /dev/null and b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/models/transformer_diffusion/__pycache__/configuration_dit_diffusion.cpython-310.pyc differ diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/models/transformer_diffusion/__pycache__/modeling_dit_diffusion.cpython-310.pyc b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/models/transformer_diffusion/__pycache__/modeling_dit_diffusion.cpython-310.pyc new file mode 100644 index 00000000..3083edaf Binary files /dev/null and b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/models/transformer_diffusion/__pycache__/modeling_dit_diffusion.cpython-310.pyc differ diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/models/transformer_diffusion/configuration_dit_diffusion.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/models/transformer_diffusion/configuration_dit_diffusion.py new file mode 100644 index 00000000..f7ba3157 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/models/transformer_diffusion/configuration_dit_diffusion.py @@ -0,0 +1,98 @@ +import os +from typing import Union, List +from transformers import PretrainedConfig + +from transformers.utils import logging +from transformers import AutoConfig, AutoModelForCausalLM +logger = logging.get_logger(__name__) + +MODEL_STRUCTURE = { + 'H': {'depth': 32, 'n_emb': 1280, 'num_heads': 16, }, + 'XL': {'depth':32, 'n_emb':1152, 'num_heads':16,}, + 'L': {'depth': 24, 'n_emb': 1024, 'num_heads': 16, }, # 400M + 'B': {'depth': 12, 'n_emb': 768, 'num_heads': 12, }, # 100M + 'S': {'depth': 12, 'n_emb': 384, 'num_heads': 6, }, +} + +class DitDiffusionPolicyConfig(PretrainedConfig): + ''' + Configuration for dit diffusion policy head + ''' + model_type = "dit_diffusion_policy" + def __init__( + self, + eval: bool = False, + action_dim: int = 14, # action dim + # output_dim: int = 14, # action dim + cond_dim: int = 1536, # the input dim of the condition + state_dim: int = 14, # the input dim of the state + prediction_horizon: int = 16, # horizon + n_obs_steps: int = 2, # number of observation steps + depth: int = 28, # number of DiT blocks + n_emb: int = 256, # embedding size + num_heads: int = 16, + mlp_ratio: int = 4.0, + time_as_cond: bool = True, + obs_as_cond: bool = True, + learn_sigma: bool = False, + model_size: str = "none", + num_inference_timesteps: int = 10, + num_queries: int = 16, + noise_samples: int = 8, + num_train_timesteps: int = 100, + is_tinyvla: bool = False, + **kwargs + ): + if model_size != "none": + depth = MODEL_STRUCTURE[model_size]['depth'] + n_emb = MODEL_STRUCTURE[model_size]['n_emb'] + num_heads = MODEL_STRUCTURE[model_size]['num_heads'] + else: + # raise ValueError("model_size show not be 'none'") + pass + # print("model_size should not be 'none'") + self.eval = eval + + self.input_dim = action_dim + self.output_dim = action_dim + self.prediction_horizon = prediction_horizon + + self.is_tinyvla = is_tinyvla + + self.cond_dim = cond_dim + self.state_dim = state_dim + + self.n_obs_steps = n_obs_steps + self.depth = depth + self.n_emb = n_emb + self.num_heads = num_heads + self.mlp_ratio = mlp_ratio + self.time_as_cond = time_as_cond + self.obs_as_cond = obs_as_cond + self.learn_sigma = learn_sigma + + self.num_inference_timesteps = num_inference_timesteps + self.num_queries = prediction_horizon + self.noise_samples = noise_samples + self.num_train_timesteps = num_train_timesteps + super().__init__(**kwargs) + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) + + # get the vision config dict if we are loading from CLIPConfig + if config_dict.get("model_type") == "llava_pythia": + config_dict = config_dict["action_head"] + + if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type: + logger.warning( + f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " + f"{cls.model_type}. This is not supported for all configurations of models and can yield errors." + ) + + return cls.from_dict(config_dict, **kwargs) + +AutoConfig.register("dit_diffusion_policy", DitDiffusionPolicyConfig) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/models/transformer_diffusion/modeling_dit_diffusion.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/models/transformer_diffusion/modeling_dit_diffusion.py new file mode 100644 index 00000000..a05c40f5 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/models/transformer_diffusion/modeling_dit_diffusion.py @@ -0,0 +1,665 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +from typing import Tuple + +import timm +import numpy as np +import logging + +import math +from typing import Tuple + +try: + from typing import Literal +except ImportError: + from typing_extensions import Literal + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint +from torch.jit import Final +from timm.models.vision_transformer import Mlp, use_fused_attn +from transformers.modeling_utils import PreTrainedModel +from transformers import AutoModel, AutoModelForCausalLM +# from timm.data import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, IMAGENET_INCEPTION_MEAN, IMAGENET_INCEPTION_STD, \ +# OPENAI_CLIP_MEAN, OPENAI_CLIP_STD +# from timm.layers import PatchEmbed, Mlp, DropPath, AttentionPoolLatent, RmsNorm, PatchDropout, SwiGLUPacked, \ +# trunc_normal_, lecun_normal_, resample_patch_embed, resample_abs_pos_embed, use_fused_attn, \ +# get_act_layer, get_norm_layer, LayerType +# from ._builder import build_model_with_cfg +# from ._manipulate import named_apply, checkpoint_seq, adapt_input_conv +# from ._registry import generate_default_cfgs, register_model, register_model_deprecations + +# __all__ = ['VisionTransformer'] # model_registry will add each entrypoint fn to this + + +_logger = logging.getLogger(__name__) + + +class Attention(nn.Module): + fused_attn: Final[bool] + + def __init__( + self, + dim: int, + num_heads: int = 8, + qkv_bias: bool = False, + qk_norm: bool = False, + attn_drop: float = 0., + proj_drop: float = 0., + norm_layer: nn.Module = nn.LayerNorm, + ) -> None: + super().__init__() + assert dim % num_heads == 0, 'dim should be divisible by num_heads' + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.scale = self.head_dim ** -0.5 + self.fused_attn = use_fused_attn() + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.q_norm = norm_layer(self.head_dim) if qk_norm else nn.Identity() + self.k_norm = norm_layer(self.head_dim) if qk_norm else nn.Identity() + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x: torch.Tensor, attn_mask=None) -> torch.Tensor: + B, N, C = x.shape + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4) + q, k, v = qkv.unbind(0) + q, k = self.q_norm(q), self.k_norm(k) + + if self.fused_attn: + x = F.scaled_dot_product_attention( + q, k, v, attn_mask=attn_mask, + dropout_p=self.attn_drop.p if self.training else 0., + ) + else: + q = q * self.scale + # attn = q @ k.transpose(-2, -1) + # if attn_mask is not None: + # attn += attn_mask + # attn = attn.softmax(dim=-1) + # attn = self.attn_drop(attn) + # x = attn @ v + attn_scores = torch.matmul(q, k.transpose(-2, -1)) + + # Add attention mask if provided + if attn_mask is not None: + attn_scores += attn_mask + + # Apply softmax to get attention weights (softmax is applied along the last dimension) + attn_weights = F.softmax(attn_scores, dim=-1) + + # Dropout on attention weights (if dropout is used) + attn_weights = self.attn_drop(attn_weights) + + # Apply attention weights to value tensor (V) + x = torch.matmul(attn_weights, v) + + x = x.transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +logger = logging.getLogger(__name__) + + +def modulate(x, shift, scale): + return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1) + + +################################################################################# +# Embedding Layers for Timesteps and Class Labels # +################################################################################# + +class TimestepEmbedder(nn.Module): + """ + Embeds scalar timesteps into vector representations. + """ + + def __init__(self, hidden_size, frequency_embedding_size=256): + super().__init__() + self.mlp = nn.Sequential( + nn.Linear(frequency_embedding_size, hidden_size, bias=True), + nn.SiLU(), + nn.Linear(hidden_size, hidden_size, bias=True), + ) + self.frequency_embedding_size = frequency_embedding_size + + @staticmethod + def timestep_embedding(t, dim, max_period=10000): + """ + Create sinusoidal timestep embeddings. + :param t: a 1-D Tensor of N indices, one per batch element. + These may be fractional. + :param dim: the dimension of the output. + :param max_period: controls the minimum frequency of the embeddings. + :return: an (N, D) Tensor of positional embeddings. + """ + # https://github.com/openai/glide-text2im/blob/main/glide_text2im/nn.py + half = dim // 2 + freqs = torch.exp( + -math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.bfloat16) / half + ).to(device=t.device) + args = t[:, None].float() * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) + return embedding.to(dtype=torch.bfloat16) + + def forward(self, t): + t_freq = self.timestep_embedding(t, self.frequency_embedding_size) + #print("@@"*50) + #print(t_freq.dtype) + t_emb = self.mlp(t_freq) + return t_emb + + +# class LabelEmbedder(nn.Module): +# """ +# Embeds class labels into vector representations. Also handles label dropout for classifier-free guidance. +# """ +# +# def __init__(self, num_classes, hidden_size, dropout_prob): +# super().__init__() +# use_cfg_embedding = dropout_prob > 0 +# self.embedding_table = nn.Embedding(num_classes + use_cfg_embedding, hidden_size) +# self.num_classes = num_classes +# self.dropout_prob = dropout_prob +# +# def token_drop(self, labels, force_drop_ids=None): +# """ +# Drops labels to enable classifier-free guidance. +# """ +# if force_drop_ids is None: +# drop_ids = torch.rand(labels.shape[0], device=labels.device) < self.dropout_prob +# else: +# drop_ids = force_drop_ids == 1 +# labels = torch.where(drop_ids, self.num_classes, labels) +# return labels +# +# def forward(self, labels, train, force_drop_ids=None): +# use_dropout = self.dropout_prob > 0 +# if (train and use_dropout) or (force_drop_ids is not None): +# labels = self.token_drop(labels, force_drop_ids) +# embeddings = self.embedding_table(labels) +# return embeddings + + +################################################################################# +# Core DiT Model # +################################################################################# + +class DiTBlock(nn.Module): + """ + A DiT block with adaptive layer norm zero (adaLN-Zero) conditioning. + """ + + def __init__(self, hidden_size, num_heads, mlp_ratio=4.0, **block_kwargs): + super().__init__() + self.norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.attn = Attention(hidden_size, num_heads=num_heads, qkv_bias=True, **block_kwargs) + self.norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + mlp_hidden_dim = int(hidden_size * mlp_ratio) + approx_gelu = lambda: nn.GELU(approximate="tanh") + self.mlp = Mlp(in_features=hidden_size, hidden_features=mlp_hidden_dim, act_layer=approx_gelu, drop=0) + self.adaLN_modulation = nn.Sequential( + nn.SiLU(), + nn.Linear(hidden_size, 6 * hidden_size, bias=True) + ) + + def forward(self, x, c, attn_mask=None): + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.adaLN_modulation(c).chunk(6, dim=1) + x = x + gate_msa.unsqueeze(1) * self.attn(modulate(self.norm1(x), shift_msa, scale_msa), attn_mask=attn_mask) # norm, scale&shift, attn, scale, + x = x + gate_mlp.unsqueeze(1) * self.mlp(modulate(self.norm2(x), shift_mlp, scale_mlp)) + return x + + +class FinalLayer(nn.Module): + """ + The final layer of DiT. + """ + + def __init__(self, hidden_size, output_dim): + super().__init__() + self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.linear = nn.Linear(hidden_size, output_dim, bias=True) + self.adaLN_modulation = nn.Sequential( + nn.SiLU(), + nn.Linear(hidden_size, 2 * hidden_size, bias=True) + ) + + def forward(self, x, c): + shift, scale = self.adaLN_modulation(c).chunk(2, dim=1) + x = modulate(self.norm_final(x), shift, scale) + x = self.linear(x) + return x + +from .configuration_dit_diffusion import DitDiffusionPolicyConfig +class DiT(PreTrainedModel): + """ + Diffusion models with a Transformer backbone. + """ + config_class = DitDiffusionPolicyConfig + def __init__( + self, + config: DitDiffusionPolicyConfig, + ): + super().__init__(config) + # compute number of tokens for main trunk and condition encoder + if config.n_obs_steps is None: + config.n_obs_steps = config.prediction_horizon + T = config.prediction_horizon + T_cond = 1 + if not config.time_as_cond: + T += 1 + T_cond -= 1 + obs_as_cond = config.cond_dim > 0 + if obs_as_cond: + assert config.time_as_cond + T_cond += config.n_obs_steps + sz = T + # mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1) + # mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0)) + # self.register_buffer("mask", mask) + self.is_tinyvla = config.is_tinyvla + if config.is_tinyvla: + self.global_1d_pool = nn.AdaptiveAvgPool1d(1) + self.norm_after_pool = nn.LayerNorm(config.cond_dim) + # self.combine = nn.Linear(cond_dim+state_dim, cond_dim) + + self.learn_sigma = config.learn_sigma + self.input_dim = config.input_dim + self.output_dim = config.output_dim * 2 if config.learn_sigma else config.output_dim + self.num_heads = config.num_heads + self.external_vision_encoder = getattr(config, 'external_vision_encoder', "None") + if self.external_vision_encoder == 'resnet': + external_obs_cond_dim = 768 + else: + external_obs_cond_dim = 0 + self.x_embedder = nn.Linear(config.input_dim, config.n_emb) + self.t_embedder = TimestepEmbedder(config.n_emb) + self.cond_obs_emb = None + if obs_as_cond: + self.cond_obs_emb = nn.Linear(config.cond_dim, config.n_emb) + + self.combine = nn.Sequential( + nn.Linear(config.cond_dim+config.state_dim+external_obs_cond_dim, 1024), + nn.ReLU(), + nn.Linear(1024, 1024), + nn.ReLU(), + nn.Linear(1024, config.cond_dim) + ) + + # Will use fixed sin-cos embedding: + self.pos_embed = nn.Parameter(torch.zeros(1, config.prediction_horizon, config.n_emb)) + + self.blocks = nn.ModuleList([ + DiTBlock(config.n_emb, config.num_heads, mlp_ratio=config.mlp_ratio) for _ in range(config.depth) + ]) + self.final_layer = FinalLayer(config.n_emb, output_dim=config.output_dim) + # self.initialize_weights() + # constants + self.T = T + self.T_cond = T_cond + self.prediction_horizon = config.prediction_horizon + self.time_as_cond = config.time_as_cond + self.action_dim = config.output_dim + self.obs_as_cond = obs_as_cond + logger.info( + "number of parameters in dit: %e", sum(p.numel() for p in self.parameters()) + ) + + from diffusers.schedulers.scheduling_ddim import DDIMScheduler + self.num_inference_timesteps = config.num_inference_timesteps + # self.proj_to_action = nn.Identity() + self.noise_scheduler = DDIMScheduler( + num_train_timesteps=config.num_train_timesteps, # 100 + beta_schedule='squaredcos_cap_v2', + clip_sample=True, + set_alpha_to_one=True, + steps_offset=0, + prediction_type='epsilon' + ) + self.num_queries = config.num_queries #16 + self.noise_samples = config.noise_samples # 1 + # self.num_inference_timesteps = config.num_inference_timesteps # 100 + + def initialize_weights(self): + # Initialize transformer layers: + def _basic_init(module): + if isinstance(module, nn.Linear): + torch.nn.init.xavier_uniform_(module.weight) + if module.bias is not None: + nn.init.constant_(module.bias, 0) + + self.apply(_basic_init) + + # Initialize (and freeze) pos_embed by sin-cos embedding: + # pos_embed = get_2d_sincos_pos_embed(self.pos_embed.shape[-1], int(self.x_embedder.num_patches ** 0.5)) + # self.pos_embed.data.copy_(torch.from_numpy(pos_embed).float().unsqueeze(0)) + nn.init.normal_(self.pos_embed, mean=0.0, std=0.02) + + # Initialize patch_embed like nn.Linear (instead of nn.Conv2d): + w = self.x_embedder.weight.data + nn.init.xavier_uniform_(w.view([w.shape[0], -1])) + nn.init.constant_(self.x_embedder.bias, 0) + + # Initialize label embedding table: + nn.init.normal_(self.cond_obs_emb.weight, mean=0.0, std=0.02) + nn.init.constant_(self.cond_obs_emb.bias, 0) + + # Initialize timestep embedding MLP: + nn.init.normal_(self.t_embedder.mlp[0].weight, std=0.02) + nn.init.normal_(self.t_embedder.mlp[2].weight, std=0.02) + + # Zero-out adaLN modulation layers in DiT blocks: + for block in self.blocks: + nn.init.constant_(block.adaLN_modulation[-1].weight, 0) + nn.init.constant_(block.adaLN_modulation[-1].bias, 0) + + # Zero-out output layers: + nn.init.constant_(self.final_layer.adaLN_modulation[-1].weight, 0) + nn.init.constant_(self.final_layer.adaLN_modulation[-1].bias, 0) + nn.init.constant_(self.final_layer.linear.weight, 0) + nn.init.constant_(self.final_layer.linear.bias, 0) + + # def unpatchify(self, x): + # """ + # x: (N, T, patch_size**2 * C) + # imgs: (N, H, W, C) + # """ + # c = self.out_channels + # p = self.x_embedder.patch_size[0] + # h = w = int(x.shape[1] ** 0.5) + # assert h * w == x.shape[1] + # + # x = x.reshape(shape=(x.shape[0], h, w, p, p, c)) + # x = torch.einsum('nhwpqc->nchpwq', x) + # imgs = x.reshape(shape=(x.shape[0], c, h * p, h * p)) + # return imgs + + def get_optim_groups(self, weight_decay: float = 1e-3): + """ + This long function is unfortunately doing something very simple and is being very defensive: + We are separating out all parameters of the models into two buckets: those that will experience + weight decay for regularization and those that won't (biases, and layernorm/embedding weights). + We are then returning the PyTorch optimizer object. + """ + + # separate out all parameters to those that will and won't experience regularizing weight decay + decay = set() + no_decay = set() + whitelist_weight_modules = (torch.nn.Linear, Attention) + blacklist_weight_modules = (torch.nn.LayerNorm, torch.nn.Embedding) + for mn, m in self.named_modules(): + for pn, p in m.named_parameters(): + fpn = "%s.%s" % (mn, pn) if mn else pn # full param name + + if pn.endswith("bias"): + # all biases will not be decayed + no_decay.add(fpn) + elif pn.startswith("bias"): + # MultiheadAttention bias starts with "bias" + no_decay.add(fpn) + elif pn.endswith("weight") and isinstance(m, whitelist_weight_modules): + # weights of whitelist modules will be weight decayed + decay.add(fpn) + elif pn.endswith("weight") and isinstance(m, blacklist_weight_modules): + # weights of blacklist modules will NOT be weight decayed + no_decay.add(fpn) + + # special case the position embedding parameter in the root GPT module as not decayed + # no_decay.add("pos_emb") + # no_decay.add("_dummy_variable") + # if self.cond_pos_emb is not None: + # no_decay.add("cond_pos_emb") + + # validate that we considered every parameter + param_dict = {pn: p for pn, p in self.named_parameters()} + inter_params = decay & no_decay + union_params = decay | no_decay + assert ( + len(inter_params) == 0 + ), "parameters %s made it into both decay/no_decay sets!" % (str(inter_params),) + assert ( + len(param_dict.keys() - union_params) == 0 + ), "parameters %s were not separated into either decay/no_decay set!" % ( + str(param_dict.keys() - union_params), + ) + + # create the pytorch optimizer object + optim_groups = [ + { + "params": [param_dict[pn] for pn in sorted(list(decay))], + "weight_decay": weight_decay, + }, + { + "params": [param_dict[pn] for pn in sorted(list(no_decay))], + "weight_decay": 0.0, + }, + ] + return optim_groups + + def configure_optimizers(self, + learning_rate: float = 1e-4, + weight_decay: float = 1e-3, + betas: Tuple[float, float] = (0.9, 0.95)): + optim_groups = self.get_optim_groups(weight_decay=weight_decay) + optimizer = torch.optim.AdamW( + optim_groups, lr=learning_rate, betas=betas + ) + return optimizer + + def forward(self, actions, hidden_states, states, is_pad, external_obs_cond=None): + """ + Forward pass for the diffusion head. + :param actions: target actions, shape [B, Ta, D] D:10 = 3+6+1 + :param hidden_states: hidden states from the llava_pythia, as the condition for the diffusion, shape [B,Tokens, D] 8 1200 1024 + :param states: robot states, shape [B, D] + :return: loss + """ + if actions is not None: # training time + B = actions.size(0) + actions = actions[:, :self.num_queries] + is_pad = is_pad[:, :self.num_queries] + num_noise_samples = self.noise_samples + # sample noise to add to actions + noise = torch.randn([num_noise_samples] + list(actions.shape), device=actions.device, + dtype=actions.dtype) # num_noise, B, Ta, D(1, 2, 16, 14) + # sample a diffusion iteration for each data point + timesteps = torch.randint( + 0, self.noise_scheduler.config.num_train_timesteps, + (B,), device=actions.device + ).long() + + timesteps, noise = timesteps.to(actions.device), noise.to(actions.device) + + # add noise to the clean actions according to the noise magnitude at each diffusion iteration + # (this is the forward diffusion process) + noisy_actions = torch.cat([self.noise_scheduler.add_noise( + actions, noise[i], timesteps) + for i in range(len(noise))], dim=0) # [num_noise_samples * B, Ta, action_dim] + + noisy_actions = noisy_actions.to(dtype=actions.dtype) + assert hidden_states.ndim == 3 + + hidden_states = hidden_states.repeat(num_noise_samples, 1, 1) + timesteps = timesteps.repeat(num_noise_samples) + is_pad = is_pad.repeat(num_noise_samples, 1) + states = states.repeat(num_noise_samples, 1) + if self.external_vision_encoder == 'resnet' and external_obs_cond is not None: + hidden_states = torch.cat([hidden_states, external_obs_cond.unsqueeze(1)], dim=2) + noise_pred = self.model_forward(noisy_actions, timesteps, global_cond=hidden_states, states=states) + noise = noise.view(noise.size(0) * noise.size(1), *noise.size()[2:]) + loss = torch.nn.functional.mse_loss(noise_pred, noise, reduction='none') + loss = (loss * ~is_pad.unsqueeze(-1)).mean() + # loss_dict['loss'] = loss + return {'loss': loss} + # return loss + else: # inference time + B = 1 + Tp = self.num_queries + action_dim = self.action_dim + if self.external_vision_encoder == 'resnet' and external_obs_cond is not None: + hidden_states = torch.cat([hidden_states, external_obs_cond.unsqueeze(1)], dim=2) + # initialize action from Guassian noise + noisy_action = torch.randn((B, Tp, action_dim)).cuda() + + naction = noisy_action.to(dtype=hidden_states.dtype) + # init scheduler + self.noise_scheduler.set_timesteps(self.num_inference_timesteps) + + for k in self.noise_scheduler.timesteps: + # predict noise + noise_pred = self.model_forward(naction, k, global_cond=hidden_states, states=states) + + # inverse diffusion step (remove noise) + naction = self.noise_scheduler.step( + model_output=noise_pred, + timestep=k, + sample=naction + ).prev_sample + + return naction + + def model_forward(self, x, t, global_cond, states): + """ + Forward pass of DiT. + x: (N, T, input_dim) + t: (N,) tensor of diffusion timesteps + global_cond: (N, n_obs_steps, D) tensor of conditions: image embeddings + """ + if self.is_tinyvla: + global_cond = self.global_1d_pool(global_cond.permute(0, 2, 1)).squeeze(-1) + global_cond = self.norm_after_pool(global_cond) + else: + global_cond = global_cond.squeeze(1) + global_cond = torch.cat([global_cond, states], dim=-1) if states is not None else global_cond + global_cond = self.combine(global_cond) + + if not torch.is_tensor(t): + # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can + t = torch.tensor([t], dtype=torch.long, device=x.device) + elif torch.is_tensor(t) and len(t.shape) == 0: + t = t[None].to(x.device) + t = t.expand(t.shape[0]) + + x = self.x_embedder(x) + self.pos_embed.to(device=x.device, dtype=x.dtype) # (N, T, D), where T = prediction_horizon + t = self.t_embedder(t) # (N, D) + if self.obs_as_cond: + global_cond = self.cond_obs_emb(global_cond) # (N, D) + # c = t + global_cond.sum(dim=1) # (N, D) + c = t + global_cond # (N, D) + for block in self.blocks: + # x = block(x, c, attn_mask=self.mask) # (N, T, D) + x = block(x, c, attn_mask=None) # (N, T, D) + x = self.final_layer(x, c) # (N, T, output_dim) + return x + + # def forward_with_cfg(self, x, t, y, cfg_scale): + # """ + # Forward pass of DiT, but also batches the unconditional forward pass for classifier-free guidance. + # """ + # # https://github.com/openai/glide-text2im/blob/main/notebooks/text2im.ipynb + # half = x[: len(x) // 2] + # combined = torch.cat([half, half], dim=0) + # model_out = self.forward(combined, t, y) + # # For exact reproducibility reasons, we apply classifier-free guidance on only + # # three channels by default. The standard approach to cfg applies it to all channels. + # # This can be done by uncommenting the following line and commenting-out the line following that. + # # eps, rest = model_out[:, :self.in_channels], model_out[:, self.in_channels:] + # eps, rest = model_out[:, :3], model_out[:, 3:] + # cond_eps, uncond_eps = torch.split(eps, len(eps) // 2, dim=0) + # half_eps = uncond_eps + cfg_scale * (cond_eps - uncond_eps) + # eps = torch.cat([half_eps, half_eps], dim=0) + # return torch.cat([eps, rest], dim=1) + + +################################################################################# +# Sine/Cosine Positional Embedding Functions # +################################################################################# +# https://github.com/facebookresearch/mae/blob/main/util/pos_embed.py + +def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False, extra_tokens=0): + """ + grid_size: int of the grid height and width + return: + pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token) + """ + grid_h = np.arange(grid_size, dtype=np.float32) + grid_w = np.arange(grid_size, dtype=np.float32) + grid = np.meshgrid(grid_w, grid_h) # here w goes first + grid = np.stack(grid, axis=0) + + grid = grid.reshape([2, 1, grid_size, grid_size]) + pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid) + if cls_token and extra_tokens > 0: + pos_embed = np.concatenate([np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0) + return pos_embed + + +def get_2d_sincos_pos_embed_from_grid(embed_dim, grid): + assert embed_dim % 2 == 0 + + # use half of dimensions to encode grid_h + emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2) + emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2) + + emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D) + return emb + + +def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): + """ + embed_dim: output dimension for each position + pos: a list of positions to be encoded: size (M,) + out: (M, D) + """ + assert embed_dim % 2 == 0 + omega = np.arange(embed_dim // 2, dtype=np.float64) + omega /= embed_dim / 2. + omega = 1. / 10000 ** omega # (D/2,) + + pos = pos.reshape(-1) # (M,) + out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product + + emb_sin = np.sin(out) # (M, D/2) + emb_cos = np.cos(out) # (M, D/2) + + emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D) + return emb + + +################################################################################# +# DiT Configs # +################################################################################# +def DiT_XH(**kwargs): + return DiT(depth=36, n_emb=1760, num_heads=32, **kwargs) +def DiT_H(**kwargs): + return DiT(depth=32, n_emb=1280, num_heads=16, **kwargs) + +def DiT_XL(**kwargs): + return DiT(depth=28, n_emb=1152, num_heads=16, **kwargs) + + +def DiT_L(**kwargs): + return DiT(depth=24, n_emb=1024, num_heads=16, **kwargs) + + +def DiT_B(**kwargs): + return DiT(depth=12, n_emb=768, num_heads=12, **kwargs) + + +def DiT_S(**kwargs): + return DiT(depth=12, n_emb=512, num_heads=6, **kwargs) + + +DiT_models = { + 'DiT-XL': DiT_XL, 'DiT-L': DiT_L, 'DiT-B': DiT_B, 'DiT-S': DiT_S, +} + +AutoModel.register(DitDiffusionPolicyConfig, DiT) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/setup.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/setup.py new file mode 100644 index 00000000..5220829a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/setup.py @@ -0,0 +1,10 @@ +from distutils.core import setup +from setuptools import find_packages + +setup( + name='policy_heads', + version='0.0.0', + packages=find_packages(), + license='MIT License', + long_description=open('README.md').read(), +) \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/util/__init__.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/util/__init__.py new file mode 100644 index 00000000..168f9979 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/util/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/util/box_ops.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/util/box_ops.py new file mode 100644 index 00000000..9c088e5b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/util/box_ops.py @@ -0,0 +1,88 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Utilities for bounding box manipulation and GIoU. +""" +import torch +from torchvision.ops.boxes import box_area + + +def box_cxcywh_to_xyxy(x): + x_c, y_c, w, h = x.unbind(-1) + b = [(x_c - 0.5 * w), (y_c - 0.5 * h), + (x_c + 0.5 * w), (y_c + 0.5 * h)] + return torch.stack(b, dim=-1) + + +def box_xyxy_to_cxcywh(x): + x0, y0, x1, y1 = x.unbind(-1) + b = [(x0 + x1) / 2, (y0 + y1) / 2, + (x1 - x0), (y1 - y0)] + return torch.stack(b, dim=-1) + + +# modified from torchvision to also return the union +def box_iou(boxes1, boxes2): + area1 = box_area(boxes1) + area2 = box_area(boxes2) + + lt = torch.max(boxes1[:, None, :2], boxes2[:, :2]) # [N,M,2] + rb = torch.min(boxes1[:, None, 2:], boxes2[:, 2:]) # [N,M,2] + + wh = (rb - lt).clamp(min=0) # [N,M,2] + inter = wh[:, :, 0] * wh[:, :, 1] # [N,M] + + union = area1[:, None] + area2 - inter + + iou = inter / union + return iou, union + + +def generalized_box_iou(boxes1, boxes2): + """ + Generalized IoU from https://giou.stanford.edu/ + + The boxes should be in [x0, y0, x1, y1] format + + Returns a [N, M] pairwise matrix, where N = len(boxes1) + and M = len(boxes2) + """ + # degenerate boxes gives inf / nan results + # so do an early check + assert (boxes1[:, 2:] >= boxes1[:, :2]).all() + assert (boxes2[:, 2:] >= boxes2[:, :2]).all() + iou, union = box_iou(boxes1, boxes2) + + lt = torch.min(boxes1[:, None, :2], boxes2[:, :2]) + rb = torch.max(boxes1[:, None, 2:], boxes2[:, 2:]) + + wh = (rb - lt).clamp(min=0) # [N,M,2] + area = wh[:, :, 0] * wh[:, :, 1] + + return iou - (area - union) / area + + +def masks_to_boxes(masks): + """Compute the bounding boxes around the provided masks + + The masks should be in format [N, H, W] where N is the number of masks, (H, W) are the spatial dimensions. + + Returns a [N, 4] tensors, with the boxes in xyxy format + """ + if masks.numel() == 0: + return torch.zeros((0, 4), device=masks.device) + + h, w = masks.shape[-2:] + + y = torch.arange(0, h, dtype=torch.float) + x = torch.arange(0, w, dtype=torch.float) + y, x = torch.meshgrid(y, x) + + x_mask = (masks * x.unsqueeze(0)) + x_max = x_mask.flatten(1).max(-1)[0] + x_min = x_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0] + + y_mask = (masks * y.unsqueeze(0)) + y_max = y_mask.flatten(1).max(-1)[0] + y_min = y_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0] + + return torch.stack([x_min, y_min, x_max, y_max], 1) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/util/misc.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/util/misc.py new file mode 100644 index 00000000..dfa9fb5b --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/util/misc.py @@ -0,0 +1,468 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Misc functions, including distributed helpers. + +Mostly copy-paste from torchvision references. +""" +import os +import subprocess +import time +from collections import defaultdict, deque +import datetime +import pickle +from packaging import version +from typing import Optional, List + +import torch +import torch.distributed as dist +from torch import Tensor + +# needed due to empty tensor bug in pytorch and torchvision 0.5 +import torchvision +if version.parse(torchvision.__version__) < version.parse('0.7'): + from torchvision.ops import _new_empty_tensor + from torchvision.ops.misc import _output_size + + +class SmoothedValue(object): + """Track a series of values and provide access to smoothed values over a + window or the global series average. + """ + + def __init__(self, window_size=20, fmt=None): + if fmt is None: + fmt = "{median:.4f} ({global_avg:.4f})" + self.deque = deque(maxlen=window_size) + self.total = 0.0 + self.count = 0 + self.fmt = fmt + + def update(self, value, n=1): + self.deque.append(value) + self.count += n + self.total += value * n + + def synchronize_between_processes(self): + """ + Warning: does not synchronize the deque! + """ + if not is_dist_avail_and_initialized(): + return + t = torch.tensor([self.count, self.total], dtype=torch.float64, device='cuda') + dist.barrier() + dist.all_reduce(t) + t = t.tolist() + self.count = int(t[0]) + self.total = t[1] + + @property + def median(self): + d = torch.tensor(list(self.deque)) + return d.median().item() + + @property + def avg(self): + d = torch.tensor(list(self.deque), dtype=torch.float32) + return d.mean().item() + + @property + def global_avg(self): + return self.total / self.count + + @property + def max(self): + return max(self.deque) + + @property + def value(self): + return self.deque[-1] + + def __str__(self): + return self.fmt.format( + median=self.median, + avg=self.avg, + global_avg=self.global_avg, + max=self.max, + value=self.value) + + +def all_gather(data): + """ + Run all_gather on arbitrary picklable data (not necessarily tensors) + Args: + data: any picklable object + Returns: + list[data]: list of data gathered from each rank + """ + world_size = get_world_size() + if world_size == 1: + return [data] + + # serialized to a Tensor + buffer = pickle.dumps(data) + storage = torch.ByteStorage.from_buffer(buffer) + tensor = torch.ByteTensor(storage).to("cuda") + + # obtain Tensor size of each rank + local_size = torch.tensor([tensor.numel()], device="cuda") + size_list = [torch.tensor([0], device="cuda") for _ in range(world_size)] + dist.all_gather(size_list, local_size) + size_list = [int(size.item()) for size in size_list] + max_size = max(size_list) + + # receiving Tensor from all ranks + # we pad the tensor because torch all_gather does not support + # gathering tensors of different shapes + tensor_list = [] + for _ in size_list: + tensor_list.append(torch.empty((max_size,), dtype=torch.uint8, device="cuda")) + if local_size != max_size: + padding = torch.empty(size=(max_size - local_size,), dtype=torch.uint8, device="cuda") + tensor = torch.cat((tensor, padding), dim=0) + dist.all_gather(tensor_list, tensor) + + data_list = [] + for size, tensor in zip(size_list, tensor_list): + buffer = tensor.cpu().numpy().tobytes()[:size] + data_list.append(pickle.loads(buffer)) + + return data_list + + +def reduce_dict(input_dict, average=True): + """ + Args: + input_dict (dict): all the values will be reduced + average (bool): whether to do average or sum + Reduce the values in the dictionary from all processes so that all processes + have the averaged results. Returns a dict with the same fields as + input_dict, after reduction. + """ + world_size = get_world_size() + if world_size < 2: + return input_dict + with torch.no_grad(): + names = [] + values = [] + # sort the keys so that they are consistent across processes + for k in sorted(input_dict.keys()): + names.append(k) + values.append(input_dict[k]) + values = torch.stack(values, dim=0) + dist.all_reduce(values) + if average: + values /= world_size + reduced_dict = {k: v for k, v in zip(names, values)} + return reduced_dict + + +class MetricLogger(object): + def __init__(self, delimiter="\t"): + self.meters = defaultdict(SmoothedValue) + self.delimiter = delimiter + + def update(self, **kwargs): + for k, v in kwargs.items(): + if isinstance(v, torch.Tensor): + v = v.item() + assert isinstance(v, (float, int)) + self.meters[k].update(v) + + def __getattr__(self, attr): + if attr in self.meters: + return self.meters[attr] + if attr in self.__dict__: + return self.__dict__[attr] + raise AttributeError("'{}' object has no attribute '{}'".format( + type(self).__name__, attr)) + + def __str__(self): + loss_str = [] + for name, meter in self.meters.items(): + loss_str.append( + "{}: {}".format(name, str(meter)) + ) + return self.delimiter.join(loss_str) + + def synchronize_between_processes(self): + for meter in self.meters.values(): + meter.synchronize_between_processes() + + def add_meter(self, name, meter): + self.meters[name] = meter + + def log_every(self, iterable, print_freq, header=None): + i = 0 + if not header: + header = '' + start_time = time.time() + end = time.time() + iter_time = SmoothedValue(fmt='{avg:.4f}') + data_time = SmoothedValue(fmt='{avg:.4f}') + space_fmt = ':' + str(len(str(len(iterable)))) + 'd' + if torch.cuda.is_available(): + log_msg = self.delimiter.join([ + header, + '[{0' + space_fmt + '}/{1}]', + 'eta: {eta}', + '{meters}', + 'time: {time}', + 'data: {data}', + 'max mem: {memory:.0f}' + ]) + else: + log_msg = self.delimiter.join([ + header, + '[{0' + space_fmt + '}/{1}]', + 'eta: {eta}', + '{meters}', + 'time: {time}', + 'data: {data}' + ]) + MB = 1024.0 * 1024.0 + for obj in iterable: + data_time.update(time.time() - end) + yield obj + iter_time.update(time.time() - end) + if i % print_freq == 0 or i == len(iterable) - 1: + eta_seconds = iter_time.global_avg * (len(iterable) - i) + eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) + if torch.cuda.is_available(): + print(log_msg.format( + i, len(iterable), eta=eta_string, + meters=str(self), + time=str(iter_time), data=str(data_time), + memory=torch.cuda.max_memory_allocated() / MB)) + else: + print(log_msg.format( + i, len(iterable), eta=eta_string, + meters=str(self), + time=str(iter_time), data=str(data_time))) + i += 1 + end = time.time() + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print('{} Total time: {} ({:.4f} s / it)'.format( + header, total_time_str, total_time / len(iterable))) + + +def get_sha(): + cwd = os.path.dirname(os.path.abspath(__file__)) + + def _run(command): + return subprocess.check_output(command, cwd=cwd).decode('ascii').strip() + sha = 'N/A' + diff = "clean" + branch = 'N/A' + try: + sha = _run(['git', 'rev-parse', 'HEAD']) + subprocess.check_output(['git', 'diff'], cwd=cwd) + diff = _run(['git', 'diff-index', 'HEAD']) + diff = "has uncommited changes" if diff else "clean" + branch = _run(['git', 'rev-parse', '--abbrev-ref', 'HEAD']) + except Exception: + pass + message = f"sha: {sha}, status: {diff}, branch: {branch}" + return message + + +def collate_fn(batch): + batch = list(zip(*batch)) + batch[0] = nested_tensor_from_tensor_list(batch[0]) + return tuple(batch) + + +def _max_by_axis(the_list): + # type: (List[List[int]]) -> List[int] + maxes = the_list[0] + for sublist in the_list[1:]: + for index, item in enumerate(sublist): + maxes[index] = max(maxes[index], item) + return maxes + + +class NestedTensor(object): + def __init__(self, tensors, mask: Optional[Tensor]): + self.tensors = tensors + self.mask = mask + + def to(self, device): + # type: (Device) -> NestedTensor # noqa + cast_tensor = self.tensors.to(device) + mask = self.mask + if mask is not None: + assert mask is not None + cast_mask = mask.to(device) + else: + cast_mask = None + return NestedTensor(cast_tensor, cast_mask) + + def decompose(self): + return self.tensors, self.mask + + def __repr__(self): + return str(self.tensors) + + +def nested_tensor_from_tensor_list(tensor_list: List[Tensor]): + # TODO make this more general + if tensor_list[0].ndim == 3: + if torchvision._is_tracing(): + # nested_tensor_from_tensor_list() does not export well to ONNX + # call _onnx_nested_tensor_from_tensor_list() instead + return _onnx_nested_tensor_from_tensor_list(tensor_list) + + # TODO make it support different-sized images + max_size = _max_by_axis([list(img.shape) for img in tensor_list]) + # min_size = tuple(min(s) for s in zip(*[img.shape for img in tensor_list])) + batch_shape = [len(tensor_list)] + max_size + b, c, h, w = batch_shape + dtype = tensor_list[0].dtype + device = tensor_list[0].device + tensor = torch.zeros(batch_shape, dtype=dtype, device=device) + mask = torch.ones((b, h, w), dtype=torch.bool, device=device) + for img, pad_img, m in zip(tensor_list, tensor, mask): + pad_img[: img.shape[0], : img.shape[1], : img.shape[2]].copy_(img) + m[: img.shape[1], :img.shape[2]] = False + else: + raise ValueError('not supported') + return NestedTensor(tensor, mask) + + +# _onnx_nested_tensor_from_tensor_list() is an implementation of +# nested_tensor_from_tensor_list() that is supported by ONNX tracing. +@torch.jit.unused +def _onnx_nested_tensor_from_tensor_list(tensor_list: List[Tensor]) -> NestedTensor: + max_size = [] + for i in range(tensor_list[0].dim()): + max_size_i = torch.max(torch.stack([img.shape[i] for img in tensor_list]).to(torch.float32)).to(torch.int64) + max_size.append(max_size_i) + max_size = tuple(max_size) + + # work around for + # pad_img[: img.shape[0], : img.shape[1], : img.shape[2]].copy_(img) + # m[: img.shape[1], :img.shape[2]] = False + # which is not yet supported in onnx + padded_imgs = [] + padded_masks = [] + for img in tensor_list: + padding = [(s1 - s2) for s1, s2 in zip(max_size, tuple(img.shape))] + padded_img = torch.nn.functional.pad(img, (0, padding[2], 0, padding[1], 0, padding[0])) + padded_imgs.append(padded_img) + + m = torch.zeros_like(img[0], dtype=torch.int, device=img.device) + padded_mask = torch.nn.functional.pad(m, (0, padding[2], 0, padding[1]), "constant", 1) + padded_masks.append(padded_mask.to(torch.bool)) + + tensor = torch.stack(padded_imgs) + mask = torch.stack(padded_masks) + + return NestedTensor(tensor, mask=mask) + + +def setup_for_distributed(is_master): + """ + This function disables printing when not in master process + """ + import builtins as __builtin__ + builtin_print = __builtin__.print + + def print(*args, **kwargs): + force = kwargs.pop('force', False) + if is_master or force: + builtin_print(*args, **kwargs) + + __builtin__.print = print + + +def is_dist_avail_and_initialized(): + if not dist.is_available(): + return False + if not dist.is_initialized(): + return False + return True + + +def get_world_size(): + if not is_dist_avail_and_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank(): + if not is_dist_avail_and_initialized(): + return 0 + return dist.get_rank() + + +def is_main_process(): + return get_rank() == 0 + + +def save_on_master(*args, **kwargs): + if is_main_process(): + torch.save(*args, **kwargs) + + +def init_distributed_mode(args): + if 'RANK' in os.environ and 'WORLD_SIZE' in os.environ: + args.rank = int(os.environ["RANK"]) + args.world_size = int(os.environ['WORLD_SIZE']) + args.gpu = int(os.environ['LOCAL_RANK']) + elif 'SLURM_PROCID' in os.environ: + args.rank = int(os.environ['SLURM_PROCID']) + args.gpu = args.rank % torch.cuda.device_count() + else: + print('Not using distributed mode') + args.distributed = False + return + + args.distributed = True + + torch.cuda.set_device(args.gpu) + args.dist_backend = 'nccl' + print('| distributed init (rank {}): {}'.format( + args.rank, args.dist_url), flush=True) + torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + torch.distributed.barrier() + setup_for_distributed(args.rank == 0) + + +@torch.no_grad() +def accuracy(output, target, topk=(1,)): + """Computes the precision@k for the specified values of k""" + if target.numel() == 0: + return [torch.zeros([], device=output.device)] + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +def interpolate(input, size=None, scale_factor=None, mode="nearest", align_corners=None): + # type: (Tensor, Optional[List[int]], Optional[float], str, Optional[bool]) -> Tensor + """ + Equivalent to nn.functional.interpolate, but with support for empty batch sizes. + This will eventually be supported natively by PyTorch, and this + class can go away. + """ + if version.parse(torchvision.__version__) < version.parse('0.7'): + if input.numel() > 0: + return torch.nn.functional.interpolate( + input, size, scale_factor, mode, align_corners + ) + + output_shape = _output_size(2, input, size, scale_factor) + output_shape = list(input.shape[:-2]) + list(output_shape) + return _new_empty_tensor(input, output_shape) + else: + return torchvision.ops.misc.interpolate(input, size, scale_factor, mode, align_corners) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/util/plot_utils.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/util/plot_utils.py new file mode 100644 index 00000000..0f24bed0 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/policy_heads/util/plot_utils.py @@ -0,0 +1,107 @@ +""" +Plotting utilities to visualize training logs. +""" +import torch +import pandas as pd +import numpy as np +import seaborn as sns +import matplotlib.pyplot as plt + +from pathlib import Path, PurePath + + +def plot_logs(logs, fields=('class_error', 'loss_bbox_unscaled', 'mAP'), ewm_col=0, log_name='log.txt'): + ''' + Function to plot specific fields from training log(s). Plots both training and test results. + + :: Inputs - logs = list containing Path objects, each pointing to individual dir with a log file + - fields = which results to plot from each log file - plots both training and test for each field. + - ewm_col = optional, which column to use as the exponential weighted smoothing of the plots + - log_name = optional, name of log file if different than default 'log.txt'. + + :: Outputs - matplotlib plots of results in fields, color coded for each log file. + - solid lines are training results, dashed lines are test results. + + ''' + func_name = "plot_utils.py::plot_logs" + + # verify logs is a list of Paths (list[Paths]) or single Pathlib object Path, + # convert single Path to list to avoid 'not iterable' error + + if not isinstance(logs, list): + if isinstance(logs, PurePath): + logs = [logs] + print(f"{func_name} info: logs param expects a list argument, converted to list[Path].") + else: + raise ValueError(f"{func_name} - invalid argument for logs parameter.\n \ + Expect list[Path] or single Path obj, received {type(logs)}") + + # Quality checks - verify valid dir(s), that every item in list is Path object, and that log_name exists in each dir + for i, dir in enumerate(logs): + if not isinstance(dir, PurePath): + raise ValueError(f"{func_name} - non-Path object in logs argument of {type(dir)}: \n{dir}") + if not dir.exists(): + raise ValueError(f"{func_name} - invalid directory in logs argument:\n{dir}") + # verify log_name exists + fn = Path(dir / log_name) + if not fn.exists(): + print(f"-> missing {log_name}. Have you gotten to Epoch 1 in training?") + print(f"--> full path of missing log file: {fn}") + return + + # load log file(s) and plot + dfs = [pd.read_json(Path(p) / log_name, lines=True) for p in logs] + + fig, axs = plt.subplots(ncols=len(fields), figsize=(16, 5)) + + for df, color in zip(dfs, sns.color_palette(n_colors=len(logs))): + for j, field in enumerate(fields): + if field == 'mAP': + coco_eval = pd.DataFrame( + np.stack(df.test_coco_eval_bbox.dropna().values)[:, 1] + ).ewm(com=ewm_col).mean() + axs[j].plot(coco_eval, c=color) + else: + df.interpolate().ewm(com=ewm_col).mean().plot( + y=[f'train_{field}', f'test_{field}'], + ax=axs[j], + color=[color] * 2, + style=['-', '--'] + ) + for ax, field in zip(axs, fields): + ax.legend([Path(p).name for p in logs]) + ax.set_title(field) + + +def plot_precision_recall(files, naming_scheme='iter'): + if naming_scheme == 'exp_id': + # name becomes exp_id + names = [f.parts[-3] for f in files] + elif naming_scheme == 'iter': + names = [f.stem for f in files] + else: + raise ValueError(f'not supported {naming_scheme}') + fig, axs = plt.subplots(ncols=2, figsize=(16, 5)) + for f, color, name in zip(files, sns.color_palette("Blues", n_colors=len(files)), names): + data = torch.load(f) + # precision is n_iou, n_points, n_cat, n_area, max_det + precision = data['precision'] + recall = data['params'].recThrs + scores = data['scores'] + # take precision for all classes, all areas and 100 detections + precision = precision[0, :, :, 0, -1].mean(1) + scores = scores[0, :, :, 0, -1].mean(1) + prec = precision.mean() + rec = data['recall'][0, :, 0, -1].mean() + print(f'{naming_scheme} {name}: mAP@50={prec * 100: 05.1f}, ' + + f'score={scores.mean():0.3f}, ' + + f'f1={2 * prec * rec / (prec + rec + 1e-8):0.3f}' + ) + axs[0].plot(recall, precision, c=color) + axs[1].plot(recall, scores, c=color) + + axs[0].set_title('Precision / Recall') + axs[0].legend(names) + axs[1].set_title('Scores / Recall') + axs[1].legend(names) + return fig, axs diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/process_data.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/process_data.py new file mode 100644 index 00000000..67e4bdd7 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/process_data.py @@ -0,0 +1,140 @@ +## 本文件用于将robotwin Challenge 2 中的hdf5数据转为TinyVLA可以直接训练的数据。 +import sys + +sys.path.append('./policy/ACT/') + +import os +import h5py +import numpy as np +import cv2 +import argparse +import json + +task_prompt = { +"place_object_scale": "Place the object onto the scale.", +"place_phone_stand": "Place phone onto stand using multi-angle desk images to determine positions and plan actions.", +"stack_blocks_three": "Move the red blocks to center of the table and stack the green block on the red block ,and stack the blue block on the green block.", +} +task_reasoning = { + "place_object_scale": 0, + "stack_blocks_three": 1 +} +all_reasoning = [ + ["Pick up the object and place the object onto the scale."], + ["Move the red blocks to center of the table and stack the green block on the red block ,and stack the blue block on the green block."], +] + +def load_hdf5(dataset_path): + ''' + 从robotwin Challenge 2 生成的 hdf5文件中读取数据 + ''' + if not os.path.isfile(dataset_path): + print(f'Dataset does not exist at \n{dataset_path}\n') + exit() + + with h5py.File(dataset_path, 'r') as root: + left_gripper, left_arm = root['/joint_action/left_gripper'][()], root['/joint_action/left_arm'][()] + right_gripper, right_arm = root['/joint_action/right_gripper'][()], root['/joint_action/right_arm'][()] + image_dict = dict() + for cam_name in root[f'/observation/'].keys(): + image_dict[cam_name] = root[f'/observation/{cam_name}/rgb'][()] + + return left_gripper, left_arm, right_gripper, right_arm, image_dict + + + +def data_transform(path, episode_num, save_path, task_name): + ''' + 将原始数据转换为 VLA 模型可以使用的格式,并保存为新的 HDF5 文件。 + ''' + begin = 0 + floders = os.listdir(path) + assert episode_num <= len(floders), "data num not enough" + + if not os.path.exists(save_path): + os.makedirs(save_path) + + for i in range(episode_num): + left_gripper_all, left_arm_all, right_gripper_all, right_arm_all, image_dict = load_hdf5( + os.path.join(path, f"episode{i}.hdf5")) + qpos = [] + actions = [] + cam_high = [] + cam_right_wrist = [] + cam_left_wrist = [] + left_arm_dim = [] + right_arm_dim = [] + + last_state = None + len_traj = left_gripper_all.shape[0]-1 # reasonging action obs的长度 + for j in range(0, left_gripper_all.shape[0]): + + left_gripper, left_arm, right_gripper, right_arm = left_gripper_all[j], left_arm_all[j], right_gripper_all[ + j], right_arm_all[j], + + if j != left_gripper_all.shape[0] - 1: + state = np.concatenate((left_arm, [left_gripper], right_arm, [right_gripper]), axis=0) # joint + + state = state.astype(np.float32) + qpos.append(state) + + camera_high_bits = image_dict['head_camera'][j] + camera_high = cv2.imdecode(np.frombuffer(camera_high_bits, np.uint8), cv2.IMREAD_COLOR) + cam_high.append(camera_high) + + camera_right_wrist_bits = image_dict['right_camera'][j] + camera_right_wrist = cv2.imdecode(np.frombuffer(camera_right_wrist_bits, np.uint8), cv2.IMREAD_COLOR) + cam_right_wrist.append(camera_right_wrist) + + camera_left_wrist_bits = image_dict['left_camera'][j] + camera_left_wrist = cv2.imdecode(np.frombuffer(camera_left_wrist_bits, np.uint8), cv2.IMREAD_COLOR) + cam_left_wrist.append(camera_left_wrist) + + if j != 0: + action = state + actions.append(action) + left_arm_dim.append(left_arm.shape[0]) + right_arm_dim.append(right_arm.shape[0]) + + hdf5path = os.path.join(save_path, f'episode_{i}.hdf5') + + with h5py.File(hdf5path, 'w') as f: + f.create_dataset('action', data=np.array(actions)) + language_raw = task_prompt[task_name].encode('utf-8') + sub_reasons = [all_reasoning[task_reasoning[task_name]][0]] * int(len_traj) + f.create_dataset('language_raw', data=np.array(language_raw)) # 增加指令 + f.create_dataset('reasoning', data=np.array(sub_reasons, dtype=object)) # 加载设定的推理 + obs = f.create_group('observations') + obs.create_dataset('qpos', data=np.array(qpos)) + obs.create_dataset('qvel', data=np.array(qpos)) # 无意义为了对齐key + obs.create_dataset('left_arm_dim', data=np.array(left_arm_dim)) + obs.create_dataset('right_arm_dim', data=np.array(right_arm_dim)) + image = obs.create_group('images') + image.create_dataset('cam_high', data=np.stack(cam_high), dtype=np.uint8) + image.create_dataset('cam_right_wrist', data=np.stack(cam_right_wrist), dtype=np.uint8) + image.create_dataset('cam_left_wrist', data=np.stack(cam_left_wrist), dtype=np.uint8) + + begin += 1 + print(f"proccess {i} success!") + + return begin + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Process some episodes.') + parser.add_argument('task_name', type=str, default='bottle_adjust', + help='The name of the task (e.g., bottle_adjust)') + parser.add_argument('setting', type=str) + parser.add_argument('expert_data_num', type=int, default=50, + help='Number of episodes to process (e.g., 50)') + + args = parser.parse_args() + + task_name = args.task_name + setting = args.setting + expert_data_num = args.expert_data_num + + data_path_name = task_name + "/" + setting + "/data" + begin = 0 + begin = data_transform(os.path.join("../../data/", data_path_name), expert_data_num, + f"data/sim-{task_name}/{setting}-{expert_data_num}",task_name) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/qwen2_vl_inference.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/qwen2_vl_inference.py new file mode 100644 index 00000000..f4086ec9 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/qwen2_vl_inference.py @@ -0,0 +1,204 @@ +import copy +import os +from transformers import Qwen2VLForConditionalGeneration, AutoTokenizer, AutoProcessor +from qwen_vl_utils import process_vision_info +from tqdm import tqdm +import h5py +import torch +import numpy as np +import cv2 +from collections import Counter +import json +RED = '\033[31m' +GREEN = '\033[32m' +YELLOW = '\033[33m' +BLUE = '\033[34m' +RESET = '\033[0m' # Reset to default color +def load_hdf5(dataset_dir, dataset_name): + dataset_path = os.path.join(dataset_dir, dataset_name) + if not os.path.isfile(dataset_path): + print(f'Dataset does not exist at \n{dataset_path}\n') + exit() + + with h5py.File(dataset_path, 'r') as root: + is_sim = root.attrs['sim'] + # qpos = root['/observations/qpos'][()] + # qvel = root['/observations/qvel'][()] + # effort = root['/observations/effort'][()] + # action = root['/action'][()] + subtask = root['/subtask'][()] + + image_dict = dict() + for cam_name in root[f'/observations/images/'].keys(): + image_dict[cam_name] = root[f'/observations/images/{cam_name}'][()] + + return image_dict, subtask +def load_model(model_path='/media/rl/HDD/data/weights/Qwen2-VL-7B-Instruct'): + #"/gpfs/private/tzb/wjj/model_param/Qwen2-VL-7B-Instruct/" + + model = Qwen2VLForConditionalGeneration.from_pretrained( + model_path, torch_dtype="auto", device_map="auto" + ) + + # We recommend enabling flash_attention_2 for better acceleration and memory saving, especially in multi-image and video scenarios. + # model = Qwen2VLForConditionalGeneration.from_pretrained( + # model_path, + # torch_dtype=torch.bfloat16, + # attn_implementation="flash_attention_2", + # device_map="auto", + # ) + + # default processer + processor = AutoProcessor.from_pretrained(model_path) + + # The default range for the number of visual tokens per image in the model is 4-16384. + # You can set min_pixels and max_pixels according to your needs, such as a token range of 256-1280, to balance performance and cost. + # min_pixels = 256*28*28 + # max_pixels = 1280*28*28 + # processor = AutoProcessor.from_pretrained("Qwen/Qwen2-VL-7B-Instruct", min_pixels=min_pixels, max_pixels=max_pixels) + return model, processor + +chat_template = [ + { + "role": "user", + "content": [ + ], + } +] +prompt = """There are four images. Please detect the objects on the table and return the objects in a list. The object names can only be one of the predefined list: []. The first image contains all objects in predefined list and the first list equals to predefined list. +Notice that the first image contains 4 objects, the second image contains 3 objects, the third image contains 2 objects and the last image only contains 1 object. So the length of answer lists must be 4,3,2,1. +Your answer must be four lists corresponding to the chosen objects for each image. +Answer example:['a','b','c','d']; ['b','c','a']; ['b','c']; ['c'] +""" +# prompt = ("There are four images and the objects in images are following []. The objects on the image is grandually picked away one by one. Please find out the order in which the objects are taken away." +# "Your answer must be a list such as [a,b,c,d].") +def model_inference(model, processor, messages): + + + # Preparation for inference + text = processor.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True + ) + image_inputs, video_inputs = process_vision_info(messages) + inputs = processor( + text=[text], + images=image_inputs, + videos=video_inputs, + padding=True, + return_tensors="pt", + ) + inputs = inputs.to("cuda") + + # Inference: Generation of the output + generated_ids = model.generate(**inputs, max_new_tokens=128) + generated_ids_trimmed = [ + out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids) + ] + output_text = processor.batch_decode( + generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False + ) + print(output_text) + results = output_text[0].split(';') + results = [eval(each.strip()) for each in results] + return results + +def filter_images_by_subtask(image_dict, subtask, OUTPUT_DIR, episode): + idxs = np.where(subtask != 0)[0] + + temp_idxs =[0] + idxs[:-1].tolist() + key_frames = [] + + for i, idx in enumerate(temp_idxs): + img = image_dict['cam_high'][idx][180:480, 200:480] + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + save_name = os.path.join(OUTPUT_DIR, f'{episode}_{i}.png') + cv2.imwrite(save_name, img) + key_frames.append(save_name) + return key_frames, idxs + +def find_missing_names_counter(a,b): + count_a = Counter(a) + count_b = Counter(b) + + missing_names = [] + for name, freq_a in count_a.items(): + freq_b = count_b.get(name, 0) + if freq_a > freq_b: + missing_count = freq_a - freq_b + missing_names.extend([name] * missing_count) + return missing_names + +def label_clean_tables(DATA_DIR, model, processor, task): + + OUTPUT_DIR = os.path.join(DATA_DIR, task, 'annotations_qwen2vl') + os.makedirs(OUTPUT_DIR, exist_ok=True) + task_path = os.path.join(DATA_DIR, task) + objs = [] + try: + with open(os.path.join(OUTPUT_DIR, 'annotations.json'), 'r') as f: + anno = json.load(f) + except Exception as e: + print(e) + anno = {} + ##########################for debug######################### + # objs = ['empty bottle', 'empty bottle', 'cup', 'mug'] + ############################################################ + with open(os.path.join(task_path, "meta.txt"), 'r', encoding='utf-8') as f: + lines = f.readlines() + for each in lines: + objs.extend(each.strip().split(',')) + # os.makedirs(os.path.join(OUTPUT_DIR, task), exist_ok=True) + episodes = os.listdir(task_path) + episodes = [episode for episode in episodes if episode.endswith('.hdf5')] + episodes = sorted(episodes, key=lambda x: int(x.split('.')[0].split('_')[-1])) + + for episode in tqdm(episodes[:10]): + if episode in anno.keys() and anno[episode]['status']: + print(f"Already processed {episode}") + continue + episode_path = os.path.join(task_path, episode) + image_dict, subtask = load_hdf5(task_path, episode) + key_frames, idxs = filter_images_by_subtask(image_dict, subtask, OUTPUT_DIR, episode.split(".")[0]) + + messages = copy.deepcopy(chat_template) + for i in range(4): + messages[0]['content'].append({ + "type": "image", + "image": os.path.join(OUTPUT_DIR, f'{episode.split(".")[0]}_{i}.png'), + }) + messages[0]['content'].append({"type": "text", "text": f""}) + messages[0]['content'][-1]['text'] = prompt.replace("[]", f"[{(','.join(objs))}]") + + results = model_inference(model, processor, messages) + + print("<<<<<<<<<<<<<<<<<>>>>>>>>>>>>>>>>>") + objects = [] + status = True + for i in range(0, len(results) - 1, 1): + res = find_missing_names_counter(results[i], results[i + 1]) + objects.append(res) + if len(res) > 1 or len(res) == 0: + print(f"{YELLOW} Detected error in {episode}: {res} {RESET}") + status = False + + objects.append(results[-1]) + print(f"The order of objects in {RED} {episode} is {objects} {RESET}") + anno[episode] = { + 'path': episode_path, + 'objects_order': objects, + 'status': status, + } + + with open(os.path.join(OUTPUT_DIR, 'annotations.json'), 'w', encoding='utf-8') as f: + json.dump(anno, f, indent=4) + +if __name__ == '__main__': + model, processor = load_model("/home/jovyan/tzb/wjj/model_param/Qwen2-VL-7B-Instruct/") + tasks = [ + # 'fold_shirt_wjj1213_meeting_room', + # 'clean_table_ljm_1217', + 'clean_table_zmj_1217_green_plate_coke_can_brown_mug_bottle', + ] + DATA_DIR = "/home/jovyan/tzb/wjj/data/aloha_bimanual/aloha_4views/" + for task in tasks: + label_clean_tables(DATA_DIR=DATA_DIR, task=task, model=model, processor=processor) \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/requirements.txt b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/requirements.txt new file mode 100644 index 00000000..ef3e5ef5 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/requirements.txt @@ -0,0 +1,216 @@ +absl-py==2.1.0 +accelerate==1.0.1 +aiofiles==23.2.1 +aiohappyeyeballs==2.4.0 +aiohttp==3.10.5 +aiosignal==1.3.1 +altair==5.3.0 +anyio==4.4.0 +appdirs==1.4.4 +argcomplete==3.3.0 +asciitree==0.3.3 +asttokens==2.4.1 +async-timeout==4.0.3 +attrs==23.2.0 +av==12.3.0 +backcall==0.2.0 +beautifulsoup4==4.12.3 +bitsandbytes==0.41.0 +cachetools==5.3.3 +catkin-pkg==1.0.0 +certifi==2024.2.2 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.29.2 +colorama==0.3.0 +contourpy==1.1.1 +cycler==0.12.1 +decorator==5.1.1 +decord==0.6.0 +deepspeed==0.9.5 +diffusers==0.11.1 +distro==1.9.0 +dm-control==1.0.14 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docutils==0.20.1 +egl-probe==1.0.2 +einops==0.6.1 +einops-exts==0.0.4 +evdev==1.7.0 +exceptiongroup==1.2.2 +executing==2.0.1 +fastapi==0.110.2 +fasteners==0.19 +ffmpy==0.3.2 +filelock==3.16.0 +fonttools==4.51.0 +frozenlist==1.4.1 +fsspec==2024.9.0 +gdown==5.2.0 +gitdb==4.0.11 +GitPython==3.1.43 +glfw==2.7.0 +google-auth==2.29.0 +google-auth-oauthlib==1.0.0 +gradio==3.35.2 +gradio_client==0.2.9 +grpcio==1.62.2 +gym==0.26.2 +gym-notices==0.0.8 +h11==0.14.0 +h5py==3.11.0 +hjson==3.1.0 +httpcore==0.17.3 +httpx==0.24.0 +huggingface-hub==0.25.2 +hydra-core==1.2.0 +idna==3.7 +imageio==2.22.0 +imageio-ffmpeg==0.4.9 +importlib_resources==6.4.5 +ipython==8.12.3 +jedi==0.19.1 +Jinja2==3.1.4 +joblib==1.4.0 +jsonschema==4.21.1 +jsonschema-specifications==2023.12.1 +kiwisolver==1.4.5 +labmaze==1.0.6 +liger_kernel==0.3.1 +linkify-it-py==2.0.3 +lit==18.1.3 +llvmlite==0.41.1 +lxml==5.2.1 +Markdown==3.6 +markdown-it-py==2.2.0 +markdown2==2.4.13 +MarkupSafe==2.1.5 +matplotlib==3.7.5 +matplotlib-inline==0.1.7 +mdit-py-plugins==0.3.3 +mdurl==0.1.2 +mpmath==1.3.0 +mujoco==2.3.7 +multidict==6.1.0 +networkx==3.1 +ninja==1.11.1.1 +numba==0.58.1 +numcodecs==0.12.1 +numpy==1.24.4 +nvidia-cublas-cu11==11.10.3.66 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu11==8.5.0.96 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu11==10.9.0.58 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu11==10.2.10.91 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu11==11.4.0.1 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu11==11.7.4.91 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu11==2.14.3 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.6.77 +nvidia-nvtx-cu11==11.7.91 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +opencv-python==4.10.0.84 +orjson==3.10.1 +packaging==24.0 +pandas==2.0.3 +parso==0.8.4 +peft==0.4.0 +pexpect==4.9.0 +pickleshare==0.7.5 +pillow==10.3.0 +pkgutil_resolve_name==1.3.10 +pluggy==1.5.0 +prompt_toolkit==3.0.47 +protobuf==3.19.6 +psutil==6.0.0 +ptyprocess==0.7.0 +pure-eval==0.2.2 +py-cpuinfo==9.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pydantic==1.10.15 +pydub==0.25.1 +pygame==2.1.2 +Pygments==2.17.2 +Pympler==1.1 +pymunk==6.2.1 +pynput==1.7.6 +PyOpenGL==3.1.7 +pyparsing==3.1.4 +pyquaternion==0.9.9 +PySocks==1.7.1 +python-dateutil==2.9.0.post0 +python-multipart==0.0.9 +python-xlib==0.33 +pytz==2024.1 +PyYAML==6.0.1 +qwen-vl-utils==0.0.8 +referencing==0.34.0 +regex==2024.4.16 +requests==2.31.0 +requests-oauthlib==2.0.0 +# Editable install with no version control (robomimic==0.3.0) +rospkg==1.5.1 +rpds-py==0.18.0 +rsa==4.9 +safetensors==0.4.3 +scikit-learn==1.2.2 +scipy==1.10.1 +semantic-version==2.10.0 +sentencepiece==0.1.99 +sentry-sdk==1.45.0 +setproctitle==1.3.3 +Shapely==1.8.4 +shortuuid==1.0.13 +six==1.16.0 +smmap==5.0.1 +sniffio==1.3.1 +snowballstemmer==2.2.0 +soupsieve==2.5 +stack-data==0.6.3 +starlette==0.37.2 +svgwrite==1.4.3 +sympy==1.12 +tensorboard==2.14.0 +tensorboard-data-server==0.7.2 +tensorboardX==2.6 +termcolor==2.4.0 +threadpoolctl==3.4.0 +tianshou==0.4.10 +timm==0.9.10 +tokenizers==0.20.1 +toolz==0.12.1 +torch==2.4.1 +torchvision +tqdm==4.66.5 +traitlets==5.14.3 +transformers==4.45.2 +triton==3.0.0 +typing_extensions==4.11.0 +tzdata==2024.1 +uc-micro-py==1.0.3 +urllib3==2.2.3 +uvicorn==0.29.0 +wandb==0.16.6 +wavedrom==2.0.3.post3 +wcwidth==0.2.13 +websockets==13.0.1 +Werkzeug==3.0.2 +yarl==1.11.1 +zarr==2.16.1 +zipp==3.20.1 diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/scripts/aloha/vla_stage2_train.sh b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/scripts/aloha/vla_stage2_train.sh new file mode 100644 index 00000000..48c7141f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/scripts/aloha/vla_stage2_train.sh @@ -0,0 +1,105 @@ +#!/bin/bash +LLM=qwen2_vl #qwen2_vl paligemma +LLM_MODEL_SIZE=2B #3B +# LLM_MODEL_SIZE=2_8B +# lora only vit and tune adapter +ACTION_HEAD=dit_diffusion_policy #act #unet_diffusion_policy dit_diffusion_policy + +#echo '1h' +#sleep 1.5h +ROOT=/data/private/joy +PRETRAIN=${ROOT}/wjj/model_param/multi_head2/${ACTION_HEAD}_results/checkpoint_all/${LLM}_${LLM_MODEL_SIZE}_pure/vanilla_aloha_${LLM}_vla_pt_f_vit/qwen2_vl_all_data_1200_align_frozen_dit_lora_substep_chunk_50/checkpoint-40000 # with substeps DIT +DIT_PRETRAIN=/data/private/policy_step_60000_2025-06-15_09-15-25.ckpt +if [ "${LLM}" == "paligemma" ]; then + echo "Using PaliGemma" + mnop=${ROOT}/wjj/model_param/PaliGemma/paligemma/pixel_224/vla-paligemma-3b-pt-224 +else + mnop=${ROOT}/Qwen2-VL-${LLM_MODEL_SIZE}-Instruct # original qwen2vl + +fi +############################################################################################################################################ +TASKNAME=your_test_task +OUTPUT=${ROOT}/dex-checkpoints/stage2/${LLM}_${LLM_MODEL_SIZE}/${TASKNAME}_Stage2_DIT_H_Stage1_1_17_using_state_correct + +if [ -d "$OUTPUT" ]; then + echo 'output exists' +else + echo '!!output not exists!!' + mkdir -p $OUTPUT +fi + +mkdir -p $OUTPUT/src +cp -r ./aloha_scripts $OUTPUT/src/ +cp -r ./scripts $OUTPUT/ +cp -r ./data_utils $OUTPUT/src/ +cp -r ./dex_vla $OUTPUT/src/ +cp -r ./policy_heads $OUTPUT/src/ + +# tinyvla set "use_reasoning with_llm_head load_pretrain using_film" false +# paligemma flash_attn False + +deepspeed --master_port 29604 --num_gpus=8 --num_nodes=1 ./train_vla.py \ + --deepspeed scripts/zero2.json \ + --using_state True \ + --use_reasoning True \ + --external_vision_encoder "None" \ + --lora_enable False \ + --action_dim 14 \ + --state_dim 14 \ + --flash_attn True \ + --chunk_size 50 \ + --lora_module "vit llm" \ + --history_images_length 1 \ + --model_pretrain $PRETRAIN \ + --load_pretrain_dit True \ + --pretrain_dit_path $DIT_PRETRAIN \ + --using_film True \ + --using_ema False \ + --policy_head_type $ACTION_HEAD \ + --policy_head_size "H" \ + --with_llm_head True \ + --image_size_stable "(320,240)" \ + --image_size_wrist "(320,240)" \ + --lora_r 64 \ + --lora_alpha 256 \ + --episode_first False \ + --task_name ${TASKNAME} \ + --model_name_or_path $mnop \ + --version v0 \ + --tune_mm_mlp_adapter True \ + --freeze_vision_tower False \ + --freeze_backbone False \ + --image_aspect_ratio pad \ + --group_by_modality_length False \ + --bf16 True \ + --output_dir $OUTPUT \ + --max_steps 60000 \ + --per_device_train_batch_size 20 \ + --gradient_accumulation_steps 1 \ + --save_strategy "steps" \ + --save_steps 10000 \ + --save_total_limit 50 \ + --learning_rate 2e-5 \ + --weight_decay 0. \ + --warmup_ratio 0.01 \ + --lr_scheduler_type "cosine" \ + --logging_steps 50 \ + --tf32 True \ + --model_max_length 2048 \ + --gradient_checkpointing True \ + --dataloader_num_workers 8 \ + --lazy_preprocess True \ + --policy_class $ACTION_HEAD \ + --concat "token_cat" \ + --report_to tensorboard \ + --logging_dir $OUTPUT/log | tee $OUTPUT/log.log + +for dir in "$OUTPUT"/*/ ; do + # 检查文件夹名称是否包含'checkpoint' + if [[ "$(basename "$dir")" == *"checkpoint"* ]]; then + cp ${mnop}/preprocessor_config.json $dir + cp ${mnop}/chat_template.json $dir + # cp $OUTPUT/non_lora_trainables.bin $dir + fi +done +echo $OUTPUT diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/scripts/aloha/vla_stage3_train.sh b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/scripts/aloha/vla_stage3_train.sh new file mode 100644 index 00000000..bc6d3bb2 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/scripts/aloha/vla_stage3_train.sh @@ -0,0 +1,118 @@ +#!/bin/bash +LLM=qwen2_vl #qwen2_vl paligemma +LLM_MODEL_SIZE=2B #3B +# LLM_MODEL_SIZE=2_8B +# lora only vit and tune adapter +ACTION_HEAD=dit_diffusion_policy #act #unet_diffusion_policy dit_diffusion_policy + +ROOT=/home/jovyan/tzb # /home/jovyan/tzb || /gpfs/private/tzb +DIT_ROOT=/home/share # /home/share || /gpfs/share/share + +#PRETRAIN=${ROOT}/wjj/model_param/multi_head2/${ACTION_HEAD}_results/checkpoint_all/${LLM}_${LLM_MODEL_SIZE}_pure/vanilla_aloha_${LLM}_vla_pt_f_vit/qwen2_vl_all_data_1200_align_frozen_dit_lora_chunk_50/checkpoint-40000 # non substeps DIT +#PRETRAIN=${ROOT}/wjj/model_param/multi_head2/${ACTION_HEAD}_results/checkpoint_all/${LLM}_${LLM_MODEL_SIZE}/vanilla_aloha_${LLM}_vla_pt_f_vit/qwen2_vl_all_data_1200_combine_constant_pretrain_DIT_H_full_param/checkpoint-60000 # with substeps DIT +#PRETRAIN=${ROOT}/wjj/model_param/multi_head2/${ACTION_HEAD}_results/checkpoint_all/${LLM}_${LLM_MODEL_SIZE}/vanilla_aloha_${LLM}_vla_pt_f_vit/qwen2_vl_4_cameras_1_12_all_data_pretrain_DiT_XH_full_param_stage_1_50/checkpoint-60000 # with substeps DIT +#PRETRAIN=${ROOT}/wjj/model_param/multi_head2/${ACTION_HEAD}_results/checkpoint_all/${LLM}_${LLM_MODEL_SIZE}/vanilla_aloha_${LLM}_vla_pt_f_vit/qwen2_3_cameras_1_17_all_data_pretrain_DiT_H_full_param_stage_1_50/checkpoint-60000 # with substeps DIT +PRETRAIN=${ROOT}/wjj/train_results/dexvla_lerobot_results/qwen2_vl_3_cameras_1_17_all_data_pretrain_6w_DiT_H_Non_EMA_full_param_stage_1_50/checkpoint-60000 # with substeps DIT + +#DIT_PRETRAIN=${DIT_ROOT}/ljm/model_param/scaledp/resnet50_with_film_nosubreason/fold_t_shirt_easy_version_all_add_clean_table_1_0_4_DiT-H_320_240_32_1e-4_numsteps_40000_sub_0_2025_01_04_17_38_19/policy_step_40000_2025-01-05_13-30-34.ckpt # non substeps DIT +DIT_PRETRAIN=${DIT_ROOT}/ljm/model_param/scaledp/resnet50_with_film_subreason/fold_t_shirt_easy_version_all_add_clean_table_1_0_4_DiT-H_320_240_32_1e-4_numsteps_40000_sub_1_2025_01_04_17_26_23/policy_step_40000_2025-01-05_12-40-45.ckpt # with substeps DIT + + +if [ "${LLM}" == "paligemma" ]; then + echo "Using PaliGemma" + mnop=${ROOT}/wjj/model_param/PaliGemma/paligemma/pixel_224/vla-paligemma-3b-pt-224 +else + mnop=${ROOT}/wjj/model_param/Qwen2-VL-${LLM_MODEL_SIZE}-Instruct +fi + +mnop=$PRETRAIN # pretrain ckpt as base + +TASKNAME=folding_two_shirts_by_drag + +OUTPUT=${ROOT}/wjj/train_results/dexvla_lerobot_results/${LLM}_${LLM_MODEL_SIZE}/${TASKNAME}_stage3_DiT_H_long +if [ -d "$OUTPUT" ]; then + echo 'output exists' +else + echo '!!output not exists!!' + mkdir -p $OUTPUT +fi + +mkdir -p $OUTPUT/src +cp -r ./aloha_scripts $OUTPUT/src/ +cp -r ./scripts $OUTPUT/ +cp -r ./data_utils $OUTPUT/src/ +cp -r ./dex_vla $OUTPUT/src/ +cp -r ./policy_heads $OUTPUT/src/ + +# tinyvla set "use_reasoning with_llm_head load_pretrain using_film" false +# paligemma flash_attn False + +deepspeed --master_port 29604 --num_gpus=8 --num_nodes=1 ./train_vla.py \ + --deepspeed scripts/zero2.json \ + --use_reasoning True \ + --lora_enable False \ + --action_dim 14 \ + --state_dim 14 \ + --flash_attn True \ + --chunk_size 50 \ + --lora_module "vit llm" \ + --home_lerobot "/home/jovyan/tzb/lerobot_data/aloha" \ + --load_pretrain False \ + --history_images_length 1 \ + --model_pretrain $PRETRAIN \ + --load_pretrain_dit False \ + --pretrain_dit_path $DIT_PRETRAIN \ + --ground_truth_reasoning False \ + --using_all_reasoning_hidden False \ + --using_film True \ + --using_ema False \ + --policy_head_type $ACTION_HEAD \ + --policy_head_size "DiT_H" \ + --with_llm_head True \ + --image_size_stable "(320,240)" \ + --image_size_wrist "(320,240)" \ + --lora_r 64 \ + --lora_alpha 256 \ + --episode_first False \ + --task_name $TASKNAME \ + --model_name_or_path $mnop \ + --version v0 \ + --tune_mm_mlp_adapter True \ + --freeze_vision_tower False \ + --freeze_backbone False \ + --image_aspect_ratio pad \ + --group_by_modality_length False \ + --bf16 True \ + --output_dir $OUTPUT \ + --max_steps 100000 \ + --per_device_train_batch_size 12 \ + --gradient_accumulation_steps 1 \ + --save_strategy "steps" \ + --save_steps 20000 \ + --save_total_limit 50 \ + --learning_rate 2e-5 \ + --weight_decay 0. \ + --warmup_ratio 0.01 \ + --lr_scheduler_type "cosine" \ + --logging_steps 50 \ + --tf32 True \ + --model_max_length 2048 \ + --gradient_checkpointing True \ + --dataloader_num_workers 8 \ + --lazy_preprocess True \ + --policy_class $ACTION_HEAD \ + --concat "token_cat" \ + --report_to tensorboard \ + --logging_dir $OUTPUT/log | tee $OUTPUT/log.log + +for dir in "$OUTPUT"/*/ ; do + # 检查文件夹名称是否包含'checkpoint' + if [[ "$(basename "$dir")" == *"checkpoint"* ]]; then + cp ${mnop}/preprocessor_config.json $dir + cp ${mnop}/chat_template.json $dir + # cp $OUTPUT/non_lora_trainables.bin $dir + fi +done + +mv ./60030.log $OUTPUT +echo $OUTPUT diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/scripts/zero2.json b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/scripts/zero2.json new file mode 100644 index 00000000..1f76836e --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/scripts/zero2.json @@ -0,0 +1,24 @@ +{ + "fp16": { + "enabled": "auto", + "loss_scale": 0, + "loss_scale_window": 1000, + "initial_scale_power": 16, + "hysteresis": 2, + "min_loss_scale": 1 + }, + "bf16": { + "enabled": "auto" + }, + "train_micro_batch_size_per_gpu": "auto", + "train_batch_size": "auto", + "gradient_accumulation_steps": "auto", + "zero_optimization": { + "stage": 2, + "overlap_comm": true, + "contiguous_gradients": true, + "sub_group_size": 1e9, + "reduce_bucket_size": "auto" + }, + "timeout": 600 +} diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/scripts/zero3.json b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/scripts/zero3.json new file mode 100644 index 00000000..dc26ee5a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/scripts/zero3.json @@ -0,0 +1,49 @@ +{ + "fp16": { + "enabled": "auto", + "loss_scale": 0, + "loss_scale_window": 1000, + "initial_scale_power": 16, + "hysteresis": 2, + "min_loss_scale": 1 + }, + "bf16": { + "enabled": "auto" + }, + "optimizer": { + "type": "AdamW", + "params": { + "lr": "auto", + "betas": "auto", + "eps": "auto", + "weight_decay": "auto" + } + }, + "zero_optimization": { + "stage": 3, + "offload_optimizer": { + "device": "none", + "pin_memory": true + }, + "offload_param": { + "device": "none", + "pin_memory": true + }, + "overlap_comm": true, + "contiguous_gradients": true, + "sub_group_size": 1e9, + "reduce_bucket_size": "auto", + "stage3_prefetch_bucket_size": "auto", + "stage3_param_persistence_threshold": "auto", + "stage3_max_live_parameters": 1e9, + "stage3_max_reuse_distance": 1e9, + "stage3_gather_16bit_weights_on_model_save": true + }, + + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 100, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false +} \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/setup.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/setup.py new file mode 100644 index 00000000..f9373217 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/setup.py @@ -0,0 +1,10 @@ +from distutils.core import setup +from setuptools import find_packages + +setup( + name='act', + version='0.0.0', + packages=find_packages(), + license='MIT License', + long_description=open('README.md').read(), +) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/torch_utils.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/torch_utils.py new file mode 100644 index 00000000..9602f797 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/torch_utils.py @@ -0,0 +1,640 @@ +""" +This file contains some PyTorch utilities. +""" +import numpy as np +import torch +import torch.optim as optim +import torch.nn.functional as F + + +def soft_update(source, target, tau): + """ + Soft update from the parameters of a @source torch module to a @target torch module + with strength @tau. The update follows target = target * (1 - tau) + source * tau. + + Args: + source (torch.nn.Module): source network to push target network parameters towards + target (torch.nn.Module): target network to update + """ + for target_param, param in zip(target.parameters(), source.parameters()): + target_param.copy_( + target_param * (1.0 - tau) + param * tau + ) + + +def hard_update(source, target): + """ + Hard update @target parameters to match @source. + + Args: + source (torch.nn.Module): source network to provide parameters + target (torch.nn.Module): target network to update parameters for + """ + for target_param, param in zip(target.parameters(), source.parameters()): + target_param.copy_(param) + + +def get_torch_device(try_to_use_cuda): + """ + Return torch device. If using cuda (GPU), will also set cudnn.benchmark to True + to optimize CNNs. + + Args: + try_to_use_cuda (bool): if True and cuda is available, will use GPU + + Returns: + device (torch.Device): device to use for models + """ + if try_to_use_cuda and torch.cuda.is_available(): + torch.backends.cudnn.benchmark = True + device = torch.device("cuda:0") + else: + device = torch.device("cpu") + return device + + +def reparameterize(mu, logvar): + """ + Reparameterize for the backpropagation of z instead of q. + This makes it so that we can backpropagate through the sampling of z from + our encoder when feeding the sampled variable to the decoder. + + (See "The reparameterization trick" section of https://arxiv.org/abs/1312.6114) + + Args: + mu (torch.Tensor): batch of means from the encoder distribution + logvar (torch.Tensor): batch of log variances from the encoder distribution + + Returns: + z (torch.Tensor): batch of sampled latents from the encoder distribution that + support backpropagation + """ + # logvar = \log(\sigma^2) = 2 * \log(\sigma) + # \sigma = \exp(0.5 * logvar) + + # clamped for numerical stability + logstd = (0.5 * logvar).clamp(-4, 15) + std = torch.exp(logstd) + + # Sample \epsilon from normal distribution + # use std to create a new tensor, so we don't have to care + # about running on GPU or not + eps = std.new(std.size()).normal_() + + # Then multiply with the standard deviation and add the mean + z = eps.mul(std).add_(mu) + + return z + + +def optimizer_from_optim_params(net_optim_params, net): + """ + Helper function to return a torch Optimizer from the optim_params + section of the config for a particular network. + + Args: + optim_params (Config): optim_params part of algo_config corresponding + to @net. This determines the optimizer that is created. + + net (torch.nn.Module): module whose parameters this optimizer will be + responsible + + Returns: + optimizer (torch.optim.Optimizer): optimizer + """ + optimizer_type = net_optim_params.get("optimizer_type", "adam") + lr = net_optim_params["learning_rate"]["initial"] + + if optimizer_type == "adam": + return optim.Adam( + params=net.parameters(), + lr=lr, + weight_decay=net_optim_params["regularization"]["L2"], + ) + elif optimizer_type == "adamw": + return optim.AdamW( + params=net.parameters(), + lr=lr, + weight_decay=net_optim_params["regularization"]["L2"], + ) + + +def lr_scheduler_from_optim_params(net_optim_params, net, optimizer): + """ + Helper function to return a LRScheduler from the optim_params + section of the config for a particular network. Returns None + if a scheduler is not needed. + + Args: + optim_params (Config): optim_params part of algo_config corresponding + to @net. This determines whether a learning rate scheduler is created. + + net (torch.nn.Module): module whose parameters this optimizer will be + responsible + + optimizer (torch.optim.Optimizer): optimizer for this net + + Returns: + lr_scheduler (torch.optim.lr_scheduler or None): learning rate scheduler + """ + lr_scheduler_type = net_optim_params["learning_rate"].get("scheduler_type", "multistep") + epoch_schedule = net_optim_params["learning_rate"]["epoch_schedule"] + + lr_scheduler = None + if len(epoch_schedule) > 0: + if lr_scheduler_type == "linear": + assert len(epoch_schedule) == 1 + end_epoch = epoch_schedule[0] + + return optim.lr_scheduler.LinearLR( + optimizer, + start_factor=1.0, + end_factor=net_optim_params["learning_rate"]["decay_factor"], + total_iters=end_epoch, + ) + elif lr_scheduler_type == "multistep": + return optim.lr_scheduler.MultiStepLR( + optimizer=optimizer, + milestones=epoch_schedule, + gamma=net_optim_params["learning_rate"]["decay_factor"], + ) + else: + raise ValueError("Invalid LR scheduler type: {}".format(lr_scheduler_type)) + + return lr_scheduler + + +def backprop_for_loss(net, optim, loss, max_grad_norm=None, retain_graph=False): + """ + Backpropagate loss and update parameters for network with + name @name. + + Args: + net (torch.nn.Module): network to update + + optim (torch.optim.Optimizer): optimizer to use + + loss (torch.Tensor): loss to use for backpropagation + + max_grad_norm (float): if provided, used to clip gradients + + retain_graph (bool): if True, graph is not freed after backward call + + Returns: + grad_norms (float): average gradient norms from backpropagation + """ + + # backprop + optim.zero_grad() + loss.backward(retain_graph=retain_graph) + + # gradient clipping + if max_grad_norm is not None: + torch.nn.utils.clip_grad_norm_(net.parameters(), max_grad_norm) + + # compute grad norms + grad_norms = 0. + for p in net.parameters(): + # only clip gradients for parameters for which requires_grad is True + if p.grad is not None: + grad_norms += p.grad.data.norm(2).pow(2).item() + + # step + optim.step() + + return grad_norms + + +def rot_6d_to_axis_angle(rot_6d): + """ + Converts tensor with rot_6d representation to axis-angle representation. + """ + rot_mat = rotation_6d_to_matrix(rot_6d) + rot = matrix_to_axis_angle(rot_mat) + return rot + + +def rot_6d_to_euler_angles(rot_6d, convention="XYZ"): + """ + Converts tensor with rot_6d representation to euler representation. + """ + rot_mat = rotation_6d_to_matrix(rot_6d) + rot = matrix_to_euler_angles(rot_mat, convention=convention) + return rot + + +def axis_angle_to_rot_6d(axis_angle): + """ + Converts tensor with rot_6d representation to axis-angle representation. + """ + rot_mat = axis_angle_to_matrix(axis_angle) + rot_6d = matrix_to_rotation_6d(rot_mat) + return rot_6d + + +def euler_angles_to_rot_6d(euler_angles, convention="XYZ"): + """ + Converts tensor with rot_6d representation to euler representation. + """ + rot_mat = euler_angles_to_matrix(euler_angles, convention="XYZ") + rot_6d = matrix_to_rotation_6d(rot_mat) + return rot_6d + + +class dummy_context_mgr(): + """ + A dummy context manager - useful for having conditional scopes (such + as @maybe_no_grad). Nothing happens in this scope. + """ + + def __enter__(self): + return None + + def __exit__(self, exc_type, exc_value, traceback): + return False + + +def maybe_no_grad(no_grad): + """ + Args: + no_grad (bool): if True, the returned context will be torch.no_grad(), otherwise + it will be a dummy context + """ + return torch.no_grad() if no_grad else dummy_context_mgr() + + +""" +The following utility functions were taken from PyTorch3D: +https://github.com/facebookresearch/pytorch3d/blob/d84f274a0822da969668d00e831870fd88327845/pytorch3d/transforms/rotation_conversions.py +""" + + +def _sqrt_positive_part(x: torch.Tensor) -> torch.Tensor: + """ + Returns torch.sqrt(torch.max(0, x)) + but with a zero subgradient where x is 0. + """ + ret = torch.zeros_like(x) + positive_mask = x > 0 + ret[positive_mask] = torch.sqrt(x[positive_mask]) + return ret + + +def quaternion_to_matrix(quaternions: torch.Tensor) -> torch.Tensor: + """ + Convert rotations given as quaternions to rotation matrices. + Args: + quaternions: quaternions with real part first, + as tensor of shape (..., 4). + Returns: + Rotation matrices as tensor of shape (..., 3, 3). + """ + r, i, j, k = torch.unbind(quaternions, -1) + # fixme[58]: `/` is not supported for operand types `float` and `Tensor`. + two_s = 2.0 / (quaternions * quaternions).sum(-1) + + o = torch.stack( + ( + 1 - two_s * (j * j + k * k), + two_s * (i * j - k * r), + two_s * (i * k + j * r), + two_s * (i * j + k * r), + 1 - two_s * (i * i + k * k), + two_s * (j * k - i * r), + two_s * (i * k - j * r), + two_s * (j * k + i * r), + 1 - two_s * (i * i + j * j), + ), + -1, + ) + return o.reshape(quaternions.shape[:-1] + (3, 3)) + + +def matrix_to_quaternion(matrix: torch.Tensor) -> torch.Tensor: + """ + Convert rotations given as rotation matrices to quaternions. + Args: + matrix: Rotation matrices as tensor of shape (..., 3, 3). + Returns: + quaternions with real part first, as tensor of shape (..., 4). + """ + if matrix.size(-1) != 3 or matrix.size(-2) != 3: + raise ValueError(f"Invalid rotation matrix shape {matrix.shape}.") + + batch_dim = matrix.shape[:-2] + m00, m01, m02, m10, m11, m12, m20, m21, m22 = torch.unbind( + matrix.reshape(batch_dim + (9,)), dim=-1 + ) + + q_abs = _sqrt_positive_part( + torch.stack( + [ + 1.0 + m00 + m11 + m22, + 1.0 + m00 - m11 - m22, + 1.0 - m00 + m11 - m22, + 1.0 - m00 - m11 + m22, + ], + dim=-1, + ) + ) + + # we produce the desired quaternion multiplied by each of r, i, j, k + quat_by_rijk = torch.stack( + [ + # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and + # `int`. + torch.stack([q_abs[..., 0] ** 2, m21 - m12, m02 - m20, m10 - m01], dim=-1), + # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and + # `int`. + torch.stack([m21 - m12, q_abs[..., 1] ** 2, m10 + m01, m02 + m20], dim=-1), + # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and + # `int`. + torch.stack([m02 - m20, m10 + m01, q_abs[..., 2] ** 2, m12 + m21], dim=-1), + # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and + # `int`. + torch.stack([m10 - m01, m20 + m02, m21 + m12, q_abs[..., 3] ** 2], dim=-1), + ], + dim=-2, + ) + + # We floor here at 0.1 but the exact level is not important; if q_abs is small, + # the candidate won't be picked. + flr = torch.tensor(0.1).to(dtype=q_abs.dtype, device=q_abs.device) + quat_candidates = quat_by_rijk / (2.0 * q_abs[..., None].max(flr)) + + # if not for numerical problems, quat_candidates[i] should be same (up to a sign), + # forall i; we pick the best-conditioned one (with the largest denominator) + + return quat_candidates[ + F.one_hot(q_abs.argmax(dim=-1), num_classes=4) > 0.5, : + ].reshape(batch_dim + (4,)) + + +def axis_angle_to_matrix(axis_angle: torch.Tensor) -> torch.Tensor: + """ + Convert rotations given as axis/angle to rotation matrices. + Args: + axis_angle: Rotations given as a vector in axis angle form, + as a tensor of shape (..., 3), where the magnitude is + the angle turned anticlockwise in radians around the + vector's direction. + Returns: + Rotation matrices as tensor of shape (..., 3, 3). + """ + return quaternion_to_matrix(axis_angle_to_quaternion(axis_angle)) + + +def matrix_to_axis_angle(matrix: torch.Tensor) -> torch.Tensor: + """ + Convert rotations given as rotation matrices to axis/angle. + Args: + matrix: Rotation matrices as tensor of shape (..., 3, 3). + Returns: + Rotations given as a vector in axis angle form, as a tensor + of shape (..., 3), where the magnitude is the angle + turned anticlockwise in radians around the vector's + direction. + """ + return quaternion_to_axis_angle(matrix_to_quaternion(matrix)) + + +def axis_angle_to_quaternion(axis_angle: torch.Tensor) -> torch.Tensor: + """ + Convert rotations given as axis/angle to quaternions. + Args: + axis_angle: Rotations given as a vector in axis angle form, + as a tensor of shape (..., 3), where the magnitude is + the angle turned anticlockwise in radians around the + vector's direction. + Returns: + quaternions with real part first, as tensor of shape (..., 4). + """ + angles = torch.norm(axis_angle, p=2, dim=-1, keepdim=True) + half_angles = angles * 0.5 + eps = 1e-6 + small_angles = angles.abs() < eps + sin_half_angles_over_angles = torch.empty_like(angles) + sin_half_angles_over_angles[~small_angles] = ( + torch.sin(half_angles[~small_angles]) / angles[~small_angles] + ) + # for x small, sin(x/2) is about x/2 - (x/2)^3/6 + # so sin(x/2)/x is about 1/2 - (x*x)/48 + sin_half_angles_over_angles[small_angles] = ( + 0.5 - (angles[small_angles] * angles[small_angles]) / 48 + ) + quaternions = torch.cat( + [torch.cos(half_angles), axis_angle * sin_half_angles_over_angles], dim=-1 + ) + return quaternions + + +def quaternion_to_axis_angle(quaternions: torch.Tensor) -> torch.Tensor: + """ + Convert rotations given as quaternions to axis/angle. + Args: + quaternions: quaternions with real part first, + as tensor of shape (..., 4). + Returns: + Rotations given as a vector in axis angle form, as a tensor + of shape (..., 3), where the magnitude is the angle + turned anticlockwise in radians around the vector's + direction. + """ + norms = torch.norm(quaternions[..., 1:], p=2, dim=-1, keepdim=True) + half_angles = torch.atan2(norms, quaternions[..., :1]) + angles = 2 * half_angles + eps = 1e-6 + small_angles = angles.abs() < eps + sin_half_angles_over_angles = torch.empty_like(angles) + sin_half_angles_over_angles[~small_angles] = ( + torch.sin(half_angles[~small_angles]) / angles[~small_angles] + ) + # for x small, sin(x/2) is about x/2 - (x/2)^3/6 + # so sin(x/2)/x is about 1/2 - (x*x)/48 + sin_half_angles_over_angles[small_angles] = ( + 0.5 - (angles[small_angles] * angles[small_angles]) / 48 + ) + return quaternions[..., 1:] / sin_half_angles_over_angles + + +def rotation_6d_to_matrix(d6: torch.Tensor) -> torch.Tensor: + """ + Converts 6D rotation representation by Zhou et al. [1] to rotation matrix + using Gram--Schmidt orthogonalization per Section B of [1]. + Args: + d6: 6D rotation representation, of size (*, 6) + Returns: + batch of rotation matrices of size (*, 3, 3) + [1] Zhou, Y., Barnes, C., Lu, J., Yang, J., & Li, H. + On the Continuity of Rotation Representations in Neural Networks. + IEEE Conference on Computer Vision and Pattern Recognition, 2019. + Retrieved from http://arxiv.org/abs/1812.07035 + """ + + a1, a2 = d6[..., :3], d6[..., 3:] + b1 = F.normalize(a1, dim=-1) + b2 = a2 - (b1 * a2).sum(-1, keepdim=True) * b1 + b2 = F.normalize(b2, dim=-1) + b3 = torch.cross(b1, b2, dim=-1) + return torch.stack((b1, b2, b3), dim=-2) + + +def matrix_to_rotation_6d(matrix: torch.Tensor) -> torch.Tensor: + """ + Converts rotation matrices to 6D rotation representation by Zhou et al. [1] + by dropping the last row. Note that 6D representation is not unique. + Args: + matrix: batch of rotation matrices of size (*, 3, 3) + Returns: + 6D rotation representation, of size (*, 6) + [1] Zhou, Y., Barnes, C., Lu, J., Yang, J., & Li, H. + On the Continuity of Rotation Representations in Neural Networks. + IEEE Conference on Computer Vision and Pattern Recognition, 2019. + Retrieved from http://arxiv.org/abs/1812.07035 + """ + batch_dim = matrix.size()[:-2] + return matrix[..., :2, :].clone().reshape(batch_dim + (6,)) + + +def matrix_to_euler_angles(matrix: torch.Tensor, convention: str) -> torch.Tensor: + """ + Convert rotations given as rotation matrices to Euler angles in radians. + + Args: + matrix: Rotation matrices as tensor of shape (..., 3, 3). + convention: Convention string of three uppercase letters. + + Returns: + Euler angles in radians as tensor of shape (..., 3). + """ + if len(convention) != 3: + raise ValueError("Convention must have 3 letters.") + if convention[1] in (convention[0], convention[2]): + raise ValueError(f"Invalid convention {convention}.") + for letter in convention: + if letter not in ("X", "Y", "Z"): + raise ValueError(f"Invalid letter {letter} in convention string.") + if matrix.size(-1) != 3 or matrix.size(-2) != 3: + raise ValueError(f"Invalid rotation matrix shape {matrix.shape}.") + i0 = _index_from_letter(convention[0]) + i2 = _index_from_letter(convention[2]) + tait_bryan = i0 != i2 + if tait_bryan: + central_angle = torch.asin( + matrix[..., i0, i2] * (-1.0 if i0 - i2 in [-1, 2] else 1.0) + ) + else: + central_angle = torch.acos(matrix[..., i0, i0]) + + o = ( + _angle_from_tan( + convention[0], convention[1], matrix[..., i2], False, tait_bryan + ), + central_angle, + _angle_from_tan( + convention[2], convention[1], matrix[..., i0, :], True, tait_bryan + ), + ) + return torch.stack(o, -1) + + +def euler_angles_to_matrix(euler_angles: torch.Tensor, convention: str) -> torch.Tensor: + """ + Convert rotations given as Euler angles in radians to rotation matrices. + + Args: + euler_angles: Euler angles in radians as tensor of shape (..., 3). + convention: Convention string of three uppercase letters from + {"X", "Y", and "Z"}. + + Returns: + Rotation matrices as tensor of shape (..., 3, 3). + """ + if euler_angles.dim() == 0 or euler_angles.shape[-1] != 3: + raise ValueError("Invalid input euler angles.") + if len(convention) != 3: + raise ValueError("Convention must have 3 letters.") + if convention[1] in (convention[0], convention[2]): + raise ValueError(f"Invalid convention {convention}.") + for letter in convention: + if letter not in ("X", "Y", "Z"): + raise ValueError(f"Invalid letter {letter} in convention string.") + matrices = [ + _axis_angle_rotation(c, e) + for c, e in zip(convention, torch.unbind(euler_angles, -1)) + ] + # return functools.reduce(torch.matmul, matrices) + return torch.matmul(torch.matmul(matrices[0], matrices[1]), matrices[2]) + + +def _index_from_letter(letter: str) -> int: + if letter == "X": + return 0 + if letter == "Y": + return 1 + if letter == "Z": + return 2 + raise ValueError("letter must be either X, Y or Z.") + + +def _angle_from_tan( + axis: str, other_axis: str, data, horizontal: bool, tait_bryan: bool +) -> torch.Tensor: + """ + Extract the first or third Euler angle from the two members of + the matrix which are positive constant times its sine and cosine. + + Args: + axis: Axis label "X" or "Y or "Z" for the angle we are finding. + other_axis: Axis label "X" or "Y or "Z" for the middle axis in the + convention. + data: Rotation matrices as tensor of shape (..., 3, 3). + horizontal: Whether we are looking for the angle for the third axis, + which means the relevant entries are in the same row of the + rotation matrix. If not, they are in the same column. + tait_bryan: Whether the first and third axes in the convention differ. + + Returns: + Euler Angles in radians for each matrix in data as a tensor + of shape (...). + """ + + i1, i2 = {"X": (2, 1), "Y": (0, 2), "Z": (1, 0)}[axis] + if horizontal: + i2, i1 = i1, i2 + even = (axis + other_axis) in ["XY", "YZ", "ZX"] + if horizontal == even: + return torch.atan2(data[..., i1], data[..., i2]) + if tait_bryan: + return torch.atan2(-data[..., i2], data[..., i1]) + return torch.atan2(data[..., i2], -data[..., i1]) + + +def _axis_angle_rotation(axis: str, angle: torch.Tensor) -> torch.Tensor: + """ + Return the rotation matrices for one of the rotations about an axis + of which Euler angles describe, for each value of the angle given. + + Args: + axis: Axis label "X" or "Y or "Z". + angle: any shape tensor of Euler angles in radians + + Returns: + Rotation matrices as tensor of shape (..., 3, 3). + """ + + cos = torch.cos(angle) + sin = torch.sin(angle) + one = torch.ones_like(angle) + zero = torch.zeros_like(angle) + + if axis == "X": + R_flat = (one, zero, zero, zero, cos, -sin, zero, sin, cos) + elif axis == "Y": + R_flat = (cos, zero, sin, zero, one, zero, -sin, zero, cos) + elif axis == "Z": + R_flat = (cos, -sin, zero, sin, cos, zero, zero, zero, one) + else: + raise ValueError("letter must be either X, Y or Z.") + + return torch.stack(R_flat, -1).reshape(angle.shape + (3, 3)) \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/train_vla.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/train_vla.py new file mode 100644 index 00000000..b1afbbf0 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/DexVLA/train_vla.py @@ -0,0 +1,405 @@ +import gc +import pickle + +import os +import time + +os.environ["TOKENIZERS_PARALLELISM"] = "false" + +os.environ['DEVICE'] = "cuda" +os.environ["WANDB_DISABLED"] = "true" + +from data_utils.dataset import load_data # data functions +from data_utils.dataset import compute_dict_mean, set_seed # helper functions +from policy_heads import * +# from data_utils.lerobot_dataset import load_data +from aloha_scripts.constants import TASK_CONFIGS +from dex_vla.utils.robot_data_processor import DexVLAProcess +# from paligemma_vla.utils.robot_data_processor import PaliGemmaVLAProcess +from transformers import AutoConfig, AutoModel, AutoProcessor +from dex_vla import DexVLATrainer +from data_utils.data_collator import * + +import IPython +e = IPython.embed +from data_utils.data_collator import DexVLADataCollatorForSupervisedDataset, PaliGemmaVLADataCollatorForSupervisedDataset +from dex_vla import model_load_utils as ml_utils +import torch +local_rank = None +from aloha_scripts.utils import * +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>parameters<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< +@dataclass +class ActionHeadArguments: + policy_head_type: str = field(default="dit_diffusion_policy") # unet_diffusion_policy + policy_head_size: str = field(default="DiT_B") # DiT_L, DiT_XL, DiT_B, DiT_S + state_dim: int = 7 + action_dim: int = 10 + + +@dataclass +class ModelArguments: + model_name_or_path: Optional[str] = field(default="facebook/opt-125m") + version: Optional[str] = field(default="v0") + model_pretrain: Optional[str] = field(default="") # pretrained model weights path + from_scratch: bool = field(default=False) + + external_vision_encoder: Optional[str] = field(default="None") + + concat: str = field(default="None") + policy_class: str = field(default="droid_diffusion") + + # with_external_vit: bool = field(default=False) + with_llm_head: bool = field(default=False) + with_text_fcs: bool = field(default=False) + only_using_input_embeddings: bool = field(default=False) # using only input embeddings + using_film: bool = field(default=False) # fusion modules + using_xattn: bool = field(default=False) # fusion modules + + using_state: bool = field(default=False) # input states into VLM + + using_channel_cat: bool = field(default=False) + using_all_reasoning_hidden: bool = field(default=False) + ground_truth_reasoning: bool = field(default=False) + + Using_EMA_Pretrain_DiT: bool = field(default=False) + + load_pretrain_dit: bool = field(default=False) # loading pretrained dit weights + pretrain_dit_path: Optional[str] = field(default=None) # path to pretrained dit weights + + freeze_policy_head: bool = field(default=False) + is_tinyvla: bool = field(default=False) + using_joint_attn: bool = field(default=False) + + # vla_model_type: Optional[str] = field(default='dex_vla') + +@dataclass +class DataArguments: + # model_name_or_path: Optional[str] = field(default="facebook/opt-125m") # equals to base model path when set load_pretrain=True + # model_pretrain: Optional[str] = field(default="") # pretrained model weights path + lazy_preprocess: bool = False + episode_first: bool = True # batchsampler will samples episode index first and then samples timesteps + select_seg_token_mask: bool = False + use_reasoning: bool = False + is_multimodal: bool = False + image_aspect_ratio: str = 'square' + task_name: str = field(default="stack_cube_2024_6_2") + skip_mirrored_data: bool = field(default=False) + chunk_size: int = field(default=16) + delta_control: bool = field(default=False) + image_size_stable: str = "480" # default 270 x 480 and pretrain may be 180 x 320 + image_size_wrist: str = "56" # specify the image size of wrist camera + history_images_length: int = 1 + home_lerobot: str = '/media/rl/HDD/data/data/aloha_data/lerobot' + +@dataclass +class TrainingArguments(transformers.TrainingArguments): + using_ema: bool = field(default=False) # whether to use ema update whole module + + local_debug: bool = field(default=False) + + cache_dir: Optional[str] = field(default=None) + optim: str = field(default="adamw_torch") + adam_beta1: float = field(default=0.9) + adam_beta2: float = field(default=0.98) + adam_epsilon: float = field(default=1e-7) + remove_unused_columns: bool = field(default=False) + + flash_attn: bool = field(default=False) + + freeze_vision_tower: bool = field(default=False) + freeze_backbone: bool = field(default=False) + tune_mm_mlp_adapter: bool = field(default=False) + resume_from_checkpoint: bool = field(default=False) + llm_loss_weight: float = field(default=1.0) + + seed: int = field(default=0) + + # logger + logging_dir: str = field(default='./logs') # TensorBoard日志的保存目录 + logging_strategy: str = field(default='steps') # 设置为`steps`表示每几步记录一次日志 + logging_steps: int = field(default=10) + + save_steps: int = field(default=10) # 每隔多少步保存一次模型 + num_train_epochs: int = field(default=3) + max_steps: int = field(default=5000) + + # validate + do_eval: bool = field(default=False) + evaluation_strategy: str = field(default="no") + eval_steps: int = field(default=200) + per_device_eval_batch_size: int = field(default=32) + + load_pretrain: bool = False + + dataloader_pin_memory: bool = False + # lora + lora_enable: bool = False + lora_module: str = "vit" + lora_task_type: str = 'CAUSAL_LM' + lora_r: int = 64 + lora_alpha: int = 256 + lora_dropout: float = 0.05 + lora_weight_path: str = "" + lora_bias: str = "none" + non_lora_lr: Optional[float] = None + group_by_modality_length: bool = field(default=False) + + model_max_length: int = field( + default=2048, + metadata={ + "help": + "Maximum sequence length. Sequences will be right padded (and possibly truncated)." + }, + ) + double_quant: bool = field( + default=True, + metadata={"help": "Compress the quantization statistics through double quantization."} + ) + quant_type: str = field( + default="nf4", + metadata={"help": "Quantization data type to use. Should be one of `fp4` or `nf4`."} + ) + bits: int = field( + default=16, + metadata={"help": "How many bits to use."} + ) + + +# <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + +def rank0_print(*args): + if local_rank == 0: + print(*args) + +def parse_param(): + global local_rank + + parser = transformers.HfArgumentParser( + (ModelArguments, DataArguments, TrainingArguments, ActionHeadArguments)) + model_args, data_args, training_args, action_head_args = parser.parse_args_into_dataclasses() + + local_rank = training_args.local_rank + compute_dtype = (torch.float16 if training_args.fp16 else (torch.bfloat16 if training_args.bf16 else torch.float32)) + + # print("##"*50) + # print(training_args.logging_dir) + + bnb_model_from_pretrained_args = {} + if training_args.bits in [4, 8]: + from transformers import BitsAndBytesConfig + bnb_model_from_pretrained_args.update(dict( + device_map={"": training_args.device}, + load_in_4bit=training_args.bits == 4, + load_in_8bit=training_args.bits == 8, + quantization_config=BitsAndBytesConfig( + load_in_4bit=training_args.bits == 4, + load_in_8bit=training_args.bits == 8, + llm_int8_skip_modules=["mm_projector"], + llm_int8_threshold=6.0, + llm_int8_has_fp16_weight=False, + bnb_4bit_compute_dtype=compute_dtype, + bnb_4bit_use_double_quant=training_args.double_quant, + bnb_4bit_quant_type=training_args.quant_type # {'fp4', 'nf4'} + ) + )) + + config = AutoConfig.from_pretrained(model_args.model_name_or_path, **asdict(action_head_args)) + if 'paligemma2' in model_args.model_name_or_path: + cond_dim = config.projection_dim + else: + cond_dim = config.hidden_size + if action_head_args.policy_head_type == 'dit_diffusion_policy': + config.policy_head_size = action_head_args.policy_head_size + config.policy_head_config = AutoConfig.for_model(model_type=config.policy_head_type, + model_size=action_head_args.policy_head_size, + cond_dim=cond_dim, action_dim=action_head_args.action_dim, + prediction_horizon=data_args.chunk_size, + state_dim=action_head_args.state_dim, + is_tinyvla=model_args.is_tinyvla, + external_vision_encoder=model_args.external_vision_encoder) + elif action_head_args.policy_head_type == 'unet_diffusion_policy': + config.policy_head_config = AutoConfig.for_model(model_type=config.policy_head_type, + global_cond_dim=cond_dim, action_dim=action_head_args.action_dim, + state_dim=action_head_args.state_dim, + is_tinyvla=model_args.is_tinyvla) + elif action_head_args.policy_head_type == 'gemma_scale_dp_policy': + config.policy_head_size = action_head_args.policy_head_size + config.policy_head_config = AutoConfig.for_model(model_type=config.policy_head_type, + model_size=action_head_args.policy_head_size, + cond_dim=cond_dim, action_dim=action_head_args.action_dim, + prediction_horizon=data_args.chunk_size, + state_dim=action_head_args.state_dim, + is_tinyvla=model_args.is_tinyvla, + external_vision_encoder=model_args.external_vision_encoder, + using_joint_attn=model_args.using_joint_attn) + else: + raise NotImplementedError(f"Unsupported policy head type {action_head_args.policy_head_type}") + # for k,v in asdict(action_head_args).items(): + # setattr(config, k, v) + setattr(config.policy_head_config, "input_dim", asdict(action_head_args)['action_dim']) + setattr(config.policy_head_config, "state_dim", asdict(action_head_args)['state_dim']) + + for k,v in asdict(model_args).items(): + setattr(config, k, v) + config.llm_loss_weight = training_args.llm_loss_weight + + # todo + # config.vision_config['image_size_wrist'] = model_args.image_size_wrist + + # config.concat = model_args.concat + if model_args.is_tinyvla: + rank0_print(f"{RED} This is TinyVLA, Please Check Both Using_film and Using_xattn equals False:Using_film {model_args.using_film}|Using_xattn {model_args.using_xattn} {RESET}") + time.sleep(1) + return model_args, data_args, training_args, action_head_args, config, bnb_model_from_pretrained_args +def train_bc(train_dataset=None, val_dataset=None, model=None, config=None, sampler_params=None, tokenizer=None, processor=None): + + compute_dtype = (torch.float16 if training_args.fp16 else (torch.bfloat16 if config['training_args'].bf16 else torch.float32)) + if config['data_args'].history_images_length > 2: + rank0_print(f"{RED} Using History and Turn to Video mode.{RESET}") + video = True + else: + video = False + if 'paligemma' in config['model_args'].model_name_or_path.lower(): + data_collator = PaliGemmaVLADataCollatorForSupervisedDataset(multimodal_processor=processor, computed_type=compute_dtype) + + else: + data_collator = DexVLADataCollatorForSupervisedDataset(multimodal_processor=processor, computed_type=compute_dtype, tokenizer=tokenizer, video=video) + # print("data loader test............") + # from torch.utils.data import DataLoader + # data_loader = DataLoader(train_dataset, batch_size=config['training_args'].per_device_train_batch_size, collate_fn=data_collator, shuffle=True) + # for batch in data_loader: + # # batch = batch.to('cuda') + # # batch = {k:v.to('cuda') for k,v in batch.items()} + # for k,v in batch.items(): + # print(k, v.dtype) + # # model(**batch) + # # time.sleep(1) + # del batch + # gc.collect() + # # exit(0) + model.config.use_cache = True + model.config.save_pretrained(config['training_args'].output_dir) + data_module = dict(train_dataset=train_dataset, + data_collator=data_collator, + eval_dataset=val_dataset + ) + trainer = DexVLATrainer(model=model, + tokenizer=tokenizer, + args=config['training_args'], + sampler_params=sampler_params, + **data_module) + + trainer.train(resume_from_checkpoint=config['training_args'].resume_from_checkpoint) + + trainer.save_state() + + model.config.use_cache = True + + if config['training_args'].lora_enable: + state_dict = ml_utils.get_peft_state_maybe_zero_3( + model.named_parameters(), config['training_args'].lora_bias + ) + non_lora_state_dict = ml_utils.get_peft_state_non_lora_maybe_zero_3( + model.named_parameters(), require_grad_only=False + ) + if config['training_args'].local_rank == 0 or config['training_args'].local_rank == -1: + model.config.save_pretrained(config['training_args'].output_dir) + model.save_pretrained(config['training_args'].output_dir, state_dict=state_dict) + torch.save(non_lora_state_dict, + os.path.join(config['training_args'].output_dir, 'non_lora_trainables.bin')) + else: + ml_utils.safe_save_model_for_hf_trainer(trainer=trainer, + output_dir=config['training_args'].output_dir) + + + +def main(all_config=None, model_config=None): + set_seed(1) + # command line parameters + training_args = all_config['training_args'].__dict__ + # get task parameters + task_config = TASK_CONFIGS[all_config['data_args'].task_name] + episode_len = task_config['episode_len'] + camera_names = task_config['camera_names'] + dataset_dir = task_config['dataset_dir'] + name_filter = task_config.get('name_filter', lambda n: True) + stats_dir = task_config.get('stats_dir', None) + sample_weights = task_config.get('sample_weights', None) + + all_config['camera_names'] = camera_names + all_config['episode_len'] = episode_len + model_config.camera_names = camera_names + # todo this is pythia's tokenizer not paligemma + # if 'pythia' in all_config['model_args'].model_name_or_path.lower(): + tokenizer = transformers.AutoTokenizer.from_pretrained( + all_config['model_args'].model_name_or_path, + ) + multimodal_processor = AutoProcessor.from_pretrained(all_config['model_args'].model_name_or_path) + # model = None + model, data_args = ml_utils.load_model(config=all_config, qwen2_vla_config=model_config, rank0_print=rank0_print, tokenizer=tokenizer) + + if 'paligemma' in all_config['model_args'].model_name_or_path.lower(): + rank0_print(f"{RED} Using PaliGemma as VLA backbone {RESET}") + image_size = all_config['model_args'].model_name_or_path.split('-')[-1] + rank0_print(f"{RED} PaliGemma using default and constant Image size{image_size}, omitting SuperParamter:[image_size_stable, image_size_wrist] {RESET}") + + vla_process = PaliGemmaVLAProcess(tokenizer=tokenizer, multimodal_processor=multimodal_processor, data_args=all_config['data_args']) + else: + rank0_print(f"{RED} Using Qwen2VL as VLA backbone {RESET}") + vla_process = DexVLAProcess(tokenizer=tokenizer, multimodal_processor=multimodal_processor, data_args=all_config['data_args'], camera_names=camera_names) + + # train_dataset, val_dataset, stats = load_data(camera_names, + # all_config['data_args'].chunk_size, + # config=all_config, + # rank0_print=rank0_print, + # policy_class=all_config['action_head_args'].policy_head_type, + # llava_pythia_process=vla_process) + + train_dataset, val_dataset, stats, sampler_params = load_data(dataset_dir_l=dataset_dir, + name_filter=name_filter, + camera_names=camera_names, + batch_size_train=all_config['training_args'].per_device_train_batch_size, + batch_size_val=all_config['training_args'].per_device_eval_batch_size, + chunk_size=all_config['data_args'].chunk_size, + skip_mirrored_data=all_config['data_args'].skip_mirrored_data, + config=all_config, + stats_dir_l=stats_dir, + rank0_print=rank0_print, + policy_class=all_config['action_head_args'].policy_head_type, + sample_weights=sample_weights, train_ratio=0.9999, return_dataset=True, llava_pythia_process=vla_process, + action_dim=all_config['action_head_args'].action_dim) + + + + # exit(0) + stats_path = os.path.join(all_config['training_args'].output_dir, f'dataset_stats.pkl') + with open(stats_path, 'wb') as f: + pickle.dump(stats, f) + + best_ckpt_info = train_bc(train_dataset=train_dataset, model=model, val_dataset=val_dataset, config=all_config, tokenizer=tokenizer, processor=multimodal_processor) + # save dataset stats + stats_path = os.path.join(all_config['training_args'].output_dir, f'dataset_stats.pkl') + with open(stats_path, 'wb') as f: + pickle.dump(stats, f) + + +if __name__ == '__main__': + model_args, data_args, training_args, action_head_args, model_config, bnb_model_from_pretrained_args = parse_param() + config = { + 'model_args':model_args, + 'data_args':data_args, + 'training_args':training_args, + 'action_head_args':action_head_args, + 'bnb_model_from_pretrained_args':bnb_model_from_pretrained_args + } + + config_dict = {k:asdict(v) if not isinstance(v, dict) else v for k,v in config.items()} + + ckpt = os.path.join(config['training_args'].output_dir, f"checkpoint-{config['training_args'].save_steps}") + if os.path.exists(ckpt): + config['training_args'].resume_from_checkpoint = True + rank0_print(f"{RED}Resuming Training............{RESET}") + main(all_config=config, model_config=model_config) + pass + + diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/README.md b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/README.md new file mode 100644 index 00000000..14814d33 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/README.md @@ -0,0 +1,163 @@ +# GO-1 Fine-tuning and Evaluation + +This README provides instructions for fine-tuning and evaluating GO-1 model, including data generation, processing, model training, and evaluation. + +## Table of Contents + +- [GO-1 Fine-tuning and Evaluation](#go-1-fine-tuning-and-evaluation) + - [Table of Contents](#table-of-contents) + - [Environment Setup](#environment-setup) + - [1. Install RoboTwin](#1-install-robotwin) + - [2. Install GO-1](#2-install-go-1) + - [Data Generation](#data-generation) + - [Data Processing](#data-processing) + - [1. Convert RoboTwin Data to HDF5](#1-convert-robotwin-data-to-hdf5) + - [2. Convert HDF5 to LeRobot Dataset](#2-convert-hdf5-to-lerobot-dataset) + - [Model Fine-tuning](#model-fine-tuning) + - [Evaluation](#evaluation) + - [Start GO-1 Server](#start-go-1-server) + - [Start RoboTwin Client](#start-robotwin-client) + - [Evaluation Results](#evaluation-results) + +## Environment Setup + +### 1. Install RoboTwin + +Create the conda environment and install the dependencies for RoboTwin according to the [RoboTwin docs](https://robotwin-platform.github.io/doc/usage/robotwin-install.html). + +Then install the extra dependencies: + +```bash +cd policy/GO1 + +conda activate RoboTwin +pip install -r requirements.txt +``` + +### 2. Install GO-1 + +Follow the instructions in the [GO-1 repo](https://github.com/OpenDriveLab/AgiBot-World?tab=readme-ov-file#getting-started--) to set up a **separate** conda environment for GO-1. + +## Data Generation + +Follow the [RoboTwin docs](https://robotwin-platform.github.io/doc/usage/collect-data.html) to generate raw data in RoboTwin format. + +Your raw data should be organized as follows: + +``` +data/ +├── task_name/ +│ ├── task_config/ +│ │ ├── data/ +│ │ │ ├── episode0.hdf5 +│ │ │ ├── episode1.hdf5 +│ │ │ └── ... +│ │ └── instructions/ +│ │ ├── episode0.json +│ │ ├── episode1.json +│ │ └── ... +``` + +## Data Processing + +### 1. Convert RoboTwin Data to HDF5 + +```bash +# Activate the RoboTwin environment +conda activate RoboTwin + +bash robotwin2hdf5.sh + +# Example: +bash robotwin2hdf5.sh beat_block_hammer demo_clean 50 +``` + +This will create processed data in the `processed_data/--` directory. + +### 2. Convert HDF5 to LeRobot Dataset + +```bash +# Activate the GO-1 environment +conda activate go1 + +# Optional: Change the LeRobot home directory +export HF_LEROBOT_HOME=/path/to/your/lerobot + +bash hdf52lerobot.sh + +# Example: +bash hdf52lerobot.sh processed_data/beat_block_hammer-demo_clean-50/ beat_block_hammer_repo +``` + +The LeRobot dataset will be saved in `/`. + +## Model Fine-tuning + +Refer to the [GO-1 repo](https://github.com/OpenDriveLab/AgiBot-World?tab=readme-ov-file#fine-tuning-on-your-own-dataset-) for detailed instructions. + + +## Evaluation + +### Start GO-1 Server + +Start the GO-1 inference server using your fine-tuned model checkpoint and data statistics: + +```bash +cd /path/to/AgiBot-World + +conda activate go1 + +python evaluate/deploy.py --model_path /path/to/your/checkpoint --data_stats_path /path/to/your/dataset_stats.json --port +``` + +The server will will listen on port `SERVER_PORT` and wait for observations. + +### Start RoboTwin Client + +The client requires a separate terminal session. We strongly recommend using `tmux` or `screen` for this process, as evaluation can take several hours to complete. + +First config the client in [deploy_policy.yml](deploy_policy.yml): + +```yaml + +host: Server IP address (default: 127.0.0.1) +port: Server port (default: 9000) +``` + +Then use the provided [script](eval.sh) to evaluate your model: + +```bash +conda activate RoboTwin + +bash eval.sh + +# Example: +bash eval.sh beat_block_hammer demo_clean go1_demo 0 0 +``` + +**Arguments:** +- `task_name` - Name of the task (*e.g.*, `beat_block_hammer`) +- `task_config` - Task configuration (*e.g.*, `demo_randomized`, `demo_clean`) +- `ckpt_setting` - Checkpoint setting name (default: `go1_demo`) +- `seed` - Random seed (default: `0`) +- `gpu_id` - GPU ID to use (default: `0`) + +Alternatively, you can set these values in [deploy_policy.yml](deploy_policy.yml). + +The evaluation results, including videos and metrics, will be saved in the `eval_result//GO1//` directory under the project root. + + +## Evaluation Results + +Following the setup in [RoboTwin2.0 Benchmark](https://robotwin-platform.github.io/leaderboard), we report the performance of GO-1 Air model and other baselines in the table below. All models are trained on the Aloha-AgileX embodiment using 50 `demo_clean` demonstrations for 3 selected tasks (`grab_roller`, `handover_mic`, `lift_pot`), and evaluated 100 times under the `demo_clean (Easy)` and `demo_randomized (Hard)` settings. Our models are fine-tuned for 10k steps. + +| Policy \ Task | Grab Roller | | Lift Pot | | Average | +| ------------- | ----------- | ------- | -------- | ------- | ---------- | +| | Easy | Hard | Easy | Hard | | +| DP | 98% | 0% | 39% | 0% | 34.25% | +| ACT | 94% | 25% | 88% | 0% | 51.25% | +| RDT | 74% | 43% | 72% | 9% | 49.5% | +| Pi0 | **96%** | 80% | 84% | **36%** | 74% | +| GO-1 Air | 86% | 94% | **94%** | 33% | 76.75% | +| GO-1 | **96%** | **96%** | **94%** | 35% | **80.25%** | + diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/__init__.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/__init__.py new file mode 100644 index 00000000..e7dd315f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/__init__.py @@ -0,0 +1 @@ +from .deploy_policy import eval, reset_model, get_model \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/deploy_policy.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/deploy_policy.py new file mode 100644 index 00000000..24526344 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/deploy_policy.py @@ -0,0 +1,110 @@ +import collections +import logging +import math +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict + +import numpy as np +import requests +import tqdm + +import json_numpy + +json_numpy.patch() + + +class GO1Client: + def __init__(self, host: str, port: int): + self.host = host + self.port = port + self.observation_window = None + + def set_language(self, instruction): + self.instruction = instruction + print(f"successfully set instruction:{instruction}") + + def predict_action(self, payload: Dict[str, Any]) -> np.ndarray: + response = requests.post( + f"http://{self.host}:{self.port}/act", json=payload, headers={"Content-Type": "application/json"} + ) + + if response.status_code == 200: + result = response.json() + action = np.array(result) + return action + else: + print(f"Request failed, status code: {response.status_code}") + print(f"Error message: {response.text}") + return None + + def update_observation_window(self, input_rgb_arr, input_state): + self.observation_window = { + "top": input_rgb_arr[0], + "left": input_rgb_arr[1], + "right": input_rgb_arr[2], + "instruction": self.instruction, + "state": input_state.reshape(1, -1), + "ctrl_freqs": np.array([30]), + } + + def get_action(self): + assert self.observation_window is not None, "update observation_window first!" + return self.predict_action(self.observation_window) + + def reset_observation_window(self): + self.instruction = None + self.observation_window = None + print("successfully unset obs and language intruction") + + +# Encode observation for the model +def encode_obs(observation): + head_img = observation["observation"]["head_camera"]["rgb"] + right_img = observation["observation"]["right_camera"]["rgb"] + left_img = observation["observation"]["left_camera"]["rgb"] + + input_rgb_arr = [head_img, right_img, left_img] + input_state = observation["joint_action"]["vector"] + + return input_rgb_arr, input_state + + +def get_model(usr_args): + return GO1Client(usr_args["host"], usr_args["port"]) + + +def get_payload(TASK_ENV): + instruction = TASK_ENV.get_instruction() + observation = TASK_ENV.get_obs() + input_rgb_arr, input_state = encode_obs(observation) + return { + "top": input_rgb_arr[0], + "left": input_rgb_arr[1], + "right": input_rgb_arr[2], + "instruction": str(instruction), + "state": input_state.reshape(1, -1), + "ctrl_freqs": np.array([15]), + } + + +def eval(TASK_ENV, model, observation): + if model.observation_window is None: + instruction = TASK_ENV.get_instruction() + model.set_language(instruction) + + input_rgb_arr, input_state = encode_obs(observation) + model.update_observation_window(input_rgb_arr, input_state) + + actions = model.get_action() + + for action in actions: + TASK_ENV.take_action(action) + observation = TASK_ENV.get_obs() + input_rgb_arr, input_state = encode_obs(observation) + model.update_observation_window(input_rgb_arr, input_state) + + +def reset_model(model): + model.reset_observation_window() diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/deploy_policy.yml b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/deploy_policy.yml new file mode 100644 index 00000000..9a64b483 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/deploy_policy.yml @@ -0,0 +1,11 @@ +# Basic experiment configuration (keep unchanged) +policy_name: GO1 +task_name: null +task_config: null +ckpt_setting: null +seed: null +instruction_type: unseen + +# GO1 Client Parameters +host: "127.0.0.1" +port: 9000 diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/eval.sh b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/eval.sh new file mode 100755 index 00000000..01bbf675 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/eval.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +policy_name=GO1 +task_name=${1} +task_config=${2} +ckpt_setting=${3:-go1_demo} +seed=${4:-0} +gpu_id=${5:-0} # default is 0 + +export CUDA_VISIBLE_DEVICES=${gpu_id} +echo -e "\033[33mgpu id (to use): ${gpu_id}\033[0m" + +cd ../.. # move to root + +PYTHONWARNINGS=ignore::UserWarning \ +python script/eval_policy.py --config policy/$policy_name/deploy_policy.yml \ + --overrides \ + --task_name ${task_name} \ + --task_config ${task_config} \ + --ckpt_setting ${ckpt_setting} \ + --seed ${seed} \ + --policy_name ${policy_name} + \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/hdf52lerobot.sh b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/hdf52lerobot.sh new file mode 100755 index 00000000..4a1ee1c9 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/hdf52lerobot.sh @@ -0,0 +1,3 @@ +data_dir=${1} +repo_id=${2} +python scripts/convert_aloha_data_to_lerobot_robotwin.py --raw_dir $data_dir --repo_id $repo_id \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/requirements.txt b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/requirements.txt new file mode 100644 index 00000000..178b498f --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/requirements.txt @@ -0,0 +1,6 @@ +h5py +PyYAML +tqdm +requests +tyro +json-numpy \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/robotwin2hdf5.sh b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/robotwin2hdf5.sh new file mode 100644 index 00000000..d414284d --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/robotwin2hdf5.sh @@ -0,0 +1,5 @@ +task_name=${1} +setting=${2} +expert_data_num=${3} + +python scripts/process_data.py $task_name $setting $expert_data_num \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/scripts/convert_aloha_data_to_lerobot_robotwin.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/scripts/convert_aloha_data_to_lerobot_robotwin.py new file mode 100755 index 00000000..80a677aa --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/scripts/convert_aloha_data_to_lerobot_robotwin.py @@ -0,0 +1,285 @@ +""" +Modified from https://github.com/RoboTwin-Platform/RoboTwin/blob/main/policy/pi0/examples/aloha_real/convert_aloha_data_to_lerobot_robotwin.py + +Script to convert Aloha hdf5 data to the LeRobot dataset v2.1 format. +""" + +import dataclasses +from pathlib import Path +import shutil +from typing import Literal + +import h5py +from lerobot.common.datasets.lerobot_dataset import HF_LEROBOT_HOME, LeRobotDataset + +import numpy as np +import torch +import tqdm +import tyro +import json +import os +import fnmatch + + +@dataclasses.dataclass(frozen=True) +class DatasetConfig: + use_videos: bool = True + tolerance_s: float = 0.0001 + image_writer_processes: int = 10 + image_writer_threads: int = 5 + video_backend: str | None = None + + +DEFAULT_DATASET_CONFIG = DatasetConfig() + + +def create_empty_dataset( + repo_id: str, + robot_type: str, + mode: Literal["video", "image"] = "video", + *, + has_velocity: bool = False, + has_effort: bool = False, + dataset_config: DatasetConfig = DEFAULT_DATASET_CONFIG, +) -> LeRobotDataset: + motors = [ + "left_waist", + "left_shoulder", + "left_elbow", + "left_forearm_roll", + "left_wrist_angle", + "left_wrist_rotate", + "left_gripper", + "right_waist", + "right_shoulder", + "right_elbow", + "right_forearm_roll", + "right_wrist_angle", + "right_wrist_rotate", + "right_gripper", + ] + + cameras = [ + "cam_high", + "cam_left_wrist", + "cam_right_wrist", + ] + + features = { + "observation.state": { + "dtype": "float32", + "shape": (len(motors),), + "names": [ + motors, + ], + }, + "action": { + "dtype": "float32", + "shape": (len(motors),), + "names": [ + motors, + ], + }, + } + + if has_velocity: + features["observation.velocity"] = { + "dtype": "float32", + "shape": (len(motors),), + "names": [ + motors, + ], + } + + if has_effort: + features["observation.effort"] = { + "dtype": "float32", + "shape": (len(motors),), + "names": [ + motors, + ], + } + + for cam in cameras: + features[f"observation.images.{cam}"] = { + "dtype": mode, + "shape": (3, 480, 640), + "names": [ + "channels", + "height", + "width", + ], + } + + if Path(HF_LEROBOT_HOME / repo_id).exists(): + shutil.rmtree(HF_LEROBOT_HOME / repo_id) + + return LeRobotDataset.create( + repo_id=repo_id, + fps=15, + robot_type=robot_type, + features=features, + use_videos=dataset_config.use_videos, + tolerance_s=dataset_config.tolerance_s, + image_writer_processes=dataset_config.image_writer_processes, + image_writer_threads=dataset_config.image_writer_threads, + video_backend=dataset_config.video_backend, + ) + + +def get_cameras(hdf5_files: list[Path]) -> list[str]: + with h5py.File(hdf5_files[0], "r") as ep: + # ignore depth channel, not currently handled + return [key for key in ep["/observations/images"].keys() if "depth" not in key] # noqa: SIM118 + + +def has_velocity(hdf5_files: list[Path]) -> bool: + with h5py.File(hdf5_files[0], "r") as ep: + return "/observations/qvel" in ep + + +def has_effort(hdf5_files: list[Path]) -> bool: + with h5py.File(hdf5_files[0], "r") as ep: + return "/observations/effort" in ep + + +def load_raw_images_per_camera(ep: h5py.File, cameras: list[str]) -> dict[str, np.ndarray]: + imgs_per_cam = {} + for camera in cameras: + uncompressed = ep[f"/observations/images/{camera}"].ndim == 4 + + if uncompressed: + # load all images in RAM + imgs_array = ep[f"/observations/images/{camera}"][:] + else: + import cv2 + + # load one compressed image after the other in RAM and uncompress + imgs_array = [] + for data in ep[f"/observations/images/{camera}"]: + data = np.frombuffer(data, np.uint8) + # img = cv2.imdecode(nparr, cv2.IMREAD_COLOR) # 解码为彩色图像 + imgs_array.append(cv2.imdecode(data, cv2.IMREAD_COLOR)) + imgs_array = np.array(imgs_array) + + imgs_per_cam[camera] = imgs_array + return imgs_per_cam + + +def load_raw_episode_data( + ep_path: Path, +) -> tuple[ + dict[str, np.ndarray], + torch.Tensor, + torch.Tensor, + torch.Tensor | None, + torch.Tensor | None, +]: + with h5py.File(ep_path, "r") as ep: + state = torch.from_numpy(ep["/observations/qpos"][:]) + action = torch.from_numpy(ep["/action"][:]) + + velocity = None + if "/observations/qvel" in ep: + velocity = torch.from_numpy(ep["/observations/qvel"][:]) + + effort = None + if "/observations/effort" in ep: + effort = torch.from_numpy(ep["/observations/effort"][:]) + + imgs_per_cam = load_raw_images_per_camera( + ep, + [ + "cam_high", + "cam_left_wrist", + "cam_right_wrist", + ], + ) + + return imgs_per_cam, state, action, velocity, effort + + +def populate_dataset( + dataset: LeRobotDataset, + hdf5_files: list[Path], + task: str, + episodes: list[int] | None = None, +) -> LeRobotDataset: + if episodes is None: + episodes = range(len(hdf5_files)) + + for ep_idx in tqdm.tqdm(episodes): + ep_path = hdf5_files[ep_idx] + + imgs_per_cam, state, action, velocity, effort = load_raw_episode_data(ep_path) + num_frames = state.shape[0] + # add prompt + dir_path = os.path.dirname(ep_path) + json_Path = f"{dir_path}/instructions.json" + + with open(json_Path, "r") as f_instr: + instruction_dict = json.load(f_instr) + instructions = instruction_dict["instructions"] + instruction = np.random.choice(instructions) + for i in range(num_frames): + frame = {"observation.state": state[i], "action": action[i]} + + for camera, img_array in imgs_per_cam.items(): + frame[f"observation.images.{camera}"] = img_array[i] + + if velocity is not None: + frame["observation.velocity"] = velocity[i] + if effort is not None: + frame["observation.effort"] = effort[i] + dataset.add_frame(frame, task=instruction) + dataset.save_episode() + + return dataset + + +def port_aloha( + raw_dir: Path, + repo_id: str, + raw_repo_id: str | None = None, + task: str = "DEBUG", + *, + episodes: list[int] | None = None, + push_to_hub: bool = False, + is_mobile: bool = False, + mode: Literal["video", "image"] = "image", + dataset_config: DatasetConfig = DEFAULT_DATASET_CONFIG, +): + if (HF_LEROBOT_HOME / repo_id).exists(): + shutil.rmtree(HF_LEROBOT_HOME / repo_id) + + if not raw_dir.exists(): + if raw_repo_id is None: + raise ValueError("raw_repo_id must be provided if raw_dir does not exist") + hdf5_files = [] + for root, _, files in os.walk(raw_dir): + for filename in fnmatch.filter(files, "*.hdf5"): + file_path = os.path.join(root, filename) + hdf5_files.append(file_path) + + dataset = create_empty_dataset( + repo_id, + robot_type="mobile_aloha" if is_mobile else "aloha", + mode=mode, + has_effort=has_effort(hdf5_files), + has_velocity=has_velocity(hdf5_files), + dataset_config=dataset_config, + ) + dataset = populate_dataset( + dataset, + hdf5_files, + task=task, + episodes=episodes, + ) + # dataset.consolidate() + + if push_to_hub: + dataset.push_to_hub() + + +if __name__ == "__main__": + tyro.cli(port_aloha) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/scripts/process_data.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/scripts/process_data.py new file mode 100644 index 00000000..eee274d2 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/GO1/scripts/process_data.py @@ -0,0 +1,179 @@ +import os +import h5py +import numpy as np +import cv2 +import argparse +import yaml +import json + + +def load_hdf5(dataset_path): + if not os.path.isfile(dataset_path): + print(f"Dataset does not exist at \n{dataset_path}\n") + exit() + + with h5py.File(dataset_path, "r") as root: + left_gripper, left_arm = ( + root["/joint_action/left_gripper"][()], + root["/joint_action/left_arm"][()], + ) + right_gripper, right_arm = ( + root["/joint_action/right_gripper"][()], + root["/joint_action/right_arm"][()], + ) + image_dict = dict() + for cam_name in root[f"/observation/"].keys(): + image_dict[cam_name] = root[f"/observation/{cam_name}/rgb"][()] + + return left_gripper, left_arm, right_gripper, right_arm, image_dict + + +def images_encoding(imgs): + encode_data = [] + padded_data = [] + max_len = 0 + for i in range(len(imgs)): + success, encoded_image = cv2.imencode(".jpg", imgs[i]) + jpeg_data = encoded_image.tobytes() + encode_data.append(jpeg_data) + max_len = max(max_len, len(jpeg_data)) + # padding + for i in range(len(imgs)): + padded_data.append(encode_data[i].ljust(max_len, b"\0")) + return encode_data, max_len + + +def get_task_config(task_name): + with open(f"./task_config/{task_name}.yml", "r", encoding="utf-8") as f: + args = yaml.load(f.read(), Loader=yaml.FullLoader) + return args + + +def data_transform(path, episode_num, save_path): + begin = 0 + floders = os.listdir(path) + # assert episode_num <= len(floders), "data num not enough" + + if not os.path.exists(save_path): + os.makedirs(save_path) + + for i in range(episode_num): + + desc_type = "seen" + instruction_data_path = os.path.join(path, "instructions", f"episode{i}.json") + with open(instruction_data_path, "r") as f_instr: + instruction_dict = json.load(f_instr) + instructions = instruction_dict[desc_type] + save_instructions_json = {"instructions": instructions} + + os.makedirs(os.path.join(save_path, f"episode_{i}"), exist_ok=True) + + with open( + os.path.join(os.path.join(save_path, f"episode_{i}"), "instructions.json"), + "w", + ) as f: + json.dump(save_instructions_json, f, indent=2) + + left_gripper_all, left_arm_all, right_gripper_all, right_arm_all, image_dict = load_hdf5( + os.path.join(path, "data", f"episode{i}.hdf5") + ) + qpos = [] + actions = [] + cam_high = [] + cam_right_wrist = [] + cam_left_wrist = [] + left_arm_dim = [] + right_arm_dim = [] + + last_state = None + for j in range(0, left_gripper_all.shape[0]): + + left_gripper, left_arm, right_gripper, right_arm = ( + left_gripper_all[j], + left_arm_all[j], + right_gripper_all[j], + right_arm_all[j], + ) + + state = np.array(left_arm.tolist() + [left_gripper] + right_arm.tolist() + [right_gripper]) # joints angle + + state = state.astype(np.float32) + + if j != left_gripper_all.shape[0] - 1: + qpos.append(state) + + camera_high_bits = image_dict["head_camera"][j] + camera_high = cv2.imdecode(np.frombuffer(camera_high_bits, np.uint8), cv2.IMREAD_COLOR) + camera_high_resized = cv2.resize(camera_high, (640, 480)) + cam_high.append(camera_high_resized) + + camera_right_wrist_bits = image_dict["right_camera"][j] + camera_right_wrist = cv2.imdecode(np.frombuffer(camera_right_wrist_bits, np.uint8), cv2.IMREAD_COLOR) + camera_right_wrist_resized = cv2.resize(camera_right_wrist, (640, 480)) + cam_right_wrist.append(camera_right_wrist_resized) + + camera_left_wrist_bits = image_dict["left_camera"][j] + camera_left_wrist = cv2.imdecode(np.frombuffer(camera_left_wrist_bits, np.uint8), cv2.IMREAD_COLOR) + camera_left_wrist_resized = cv2.resize(camera_left_wrist, (640, 480)) + cam_left_wrist.append(camera_left_wrist_resized) + + if j != 0: + action = state + actions.append(action) + left_arm_dim.append(left_arm.shape[0]) + right_arm_dim.append(right_arm.shape[0]) + + hdf5path = os.path.join(save_path, f"episode_{i}/episode_{i}.hdf5") + + with h5py.File(hdf5path, "w") as f: + f.create_dataset("action", data=np.array(actions)) + obs = f.create_group("observations") + obs.create_dataset("qpos", data=np.array(qpos)) + obs.create_dataset("left_arm_dim", data=np.array(left_arm_dim)) + obs.create_dataset("right_arm_dim", data=np.array(right_arm_dim)) + image = obs.create_group("images") + cam_high_enc, len_high = images_encoding(cam_high) + cam_right_wrist_enc, len_right = images_encoding(cam_right_wrist) + cam_left_wrist_enc, len_left = images_encoding(cam_left_wrist) + image.create_dataset("cam_high", data=cam_high_enc, dtype=f"S{len_high}") + image.create_dataset("cam_right_wrist", data=cam_right_wrist_enc, dtype=f"S{len_right}") + image.create_dataset("cam_left_wrist", data=cam_left_wrist_enc, dtype=f"S{len_left}") + + begin += 1 + print(f"proccess {i} success!") + + return begin + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Process some episodes.") + parser.add_argument( + "task_name", + type=str, + default="beat_block_hammer", + help="The name of the task (e.g., beat_block_hammer)", + ) + parser.add_argument("setting", type=str) + parser.add_argument( + "expert_data_num", + type=int, + default=50, + help="Number of episodes to process (e.g., 50)", + ) + args = parser.parse_args() + + task_name = args.task_name + setting = args.setting + expert_data_num = args.expert_data_num + + load_dir = os.path.join("../../data", str(task_name), str(setting)) + + begin = 0 + print(f"read data from path:{load_dir}") + + target_dir = f"processed_data/{task_name}-{setting}-{expert_data_num}" + begin = data_transform( + load_dir, + expert_data_num, + target_dir, + ) diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/LLaVA-VLA/deploy_policy.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/LLaVA-VLA/deploy_policy.py new file mode 100644 index 00000000..579e53ba --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/LLaVA-VLA/deploy_policy.py @@ -0,0 +1,88 @@ +from .llava_vla_model import * +import argparse +import numpy as np +from PIL import Image + +EXPECTED_ACTION_DIM = 70 +NUM_ACTIONS = 5 +ACTION_DIM = 14 + +import torch +import torchvision.transforms as T +from PIL import Image +import os + + +def encode_obs(observation): + input_rgb_arr = [ + observation["observation"]["left_camera"]["rgb"], + observation["observation"]["right_camera"]["rgb"], + observation["observation"]["head_camera"]["rgb"], + observation["observation"]["front_camera"]["rgb"], + ] + input_state = observation["joint_action"]["vector"] + + return input_rgb_arr, input_state + + +def combined_images(images): + + images = [np.array(img) if not isinstance(img, np.ndarray) else img for img in images] + + height, width, channels = images[0].shape + + if channels != 3: + raise ValueError(f"Expected 3 channels (RGB/BGR), got {channels}") + + + images = [(img * 255).astype(np.uint8) if img.max() <= 1.0 else img.astype(np.uint8) for img in images] + images = [img[:, :, ::-1] if img[0, 0, 2] > img[0, 0, 0] else img for img in images] + stitched = np.zeros((2 * height, 2 * width, 3), dtype=np.uint8) + + # images[0] (left_camera) + stitched[0:height, 0:width] = images[0] + # images[1] (right_camera) + stitched[0:height, width:2*width] = images[1] + # images[2] (head_camera) + stitched[height:2*height, 0:width] = images[2] + # images[3] (front_camera) + stitched[height:2*height, width:2*width] = images[3] + + # RGB + combine_image = Image.fromarray(stitched) + + return combine_image + +def get_model(usr_args): + if isinstance(usr_args, dict): + usr_args = argparse.Namespace(**usr_args) + return LLaVA(usr_args) + +def eval(TASK_ENV, model, observation): + + instruction = TASK_ENV.get_instruction() + model.set_language(instruction) + input_rgb_arr, input_state = encode_obs(observation) + + image=combined_images(input_rgb_arr) + input_ids,images=model.compose_robot_input(image,instruction,input_state) + actions = model.get_action(input_ids, images) + + print("Action shape before reshape:", actions.shape) + + if actions.shape == (70,): + actions = actions.reshape(5, 14) + + for i in range(actions.shape[0]): + action = actions[i] + TASK_ENV.take_action(action) + observation = TASK_ENV.get_obs() + input_rgb_arr, input_state = encode_obs(observation) + print(f"input_state_after_action (step {i}):", input_state) + + return observation + + +def reset_model(model): + # Clean the model cache at the beginning of every evaluation episode, such as the observation window + pass \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/LLaVA-VLA/deploy_policy.yaml b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/LLaVA-VLA/deploy_policy.yaml new file mode 100644 index 00000000..220105ba --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/LLaVA-VLA/deploy_policy.yaml @@ -0,0 +1,15 @@ +policy_name: LLaVA-VLA +task_name: lift_pot +task_config: clean +ckpt_setting: clean +seed: 4 +instruction_type: seen +policy_conda_env: null +expert_data_num: 50 +model_path: /yourpath/RoboTwin/policy/LLaVA-VLA/checkpoint +action_stat: /yourpath/RoboTwin/policy/LLaVA-VLA/obs_statistics/your_obs_statistics.yaml +model_base: null +temperature: 0.0 +top_p: null +num_beams: 1 +max_new_tokens: 128 \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/LLaVA-VLA/eval.sh b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/LLaVA-VLA/eval.sh new file mode 100644 index 00000000..30fa7be2 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/LLaVA-VLA/eval.sh @@ -0,0 +1,9 @@ +#!/bin/bash +policy_name=LLaVA-VLA +gpu_id=${1} + +export PYTHONPATH=/yourpath/RoboTwin/policy/LLaVA-VLA:$PYTHONPATH +export CUDA_VISIBLE_DEVICES=${gpu_id} +echo -e "\033[33mgpu id (to use): ${gpu_id}\033[0m" +PYTHONWARNINGS=ignore::UserWarning \ +python /yourpath/RoboTwin/script/eval_policy.py --config policy/$policy_name/deploy_policy.yml \ \ No newline at end of file diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/LLaVA-VLA/llava/__init__.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/LLaVA-VLA/llava/__init__.py new file mode 100644 index 00000000..d8f73cc3 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/LLaVA-VLA/llava/__init__.py @@ -0,0 +1 @@ +from .model import LlavaLlamaForCausalLM, AsrLlavaForCausalLM diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/LLaVA-VLA/llava/action_tokenizer.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/LLaVA-VLA/llava/action_tokenizer.py new file mode 100644 index 00000000..c9f7f514 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/LLaVA-VLA/llava/action_tokenizer.py @@ -0,0 +1,289 @@ +""" +action_tokenizer.py + +Extension class; wraps base LLM/VLM tokenizer with logic to discretize and tokenize continuous robot actions. +""" + +from typing import List, Union + +import numpy as np +from transformers import PreTrainedTokenizerBase +from pathlib import Path +import yaml +import json +import torch + +class ActionTokenizer: + def __init__( + self, tokenizer: PreTrainedTokenizerBase, bins: int = 256, min_action: int = -1, max_action: int = 1, use_norm_bins: bool = False + ) -> None: + """ + Discretizes continuous robot actions into N bins per dimension and maps to the least used tokens. + + NOTE =>> by default, assumes a BPE-style tokenizer akin to the LlamaTokenizer, where *the least used tokens* + appear at the end of the vocabulary! + + :param tokenizer: Base LLM/VLM tokenizer to extend. + :param bins: Number of bins for each continuous value; we'll adopt a uniform binning strategy. + :param min_action: Minimum action value (for clipping, setting lower bound on bin interval). + :param max_action: Maximum action value (for clipping, setting upper bound on bin interval). + """ + self.tokenizer, self.n_bins, self.min_action, self.max_action = tokenizer, bins, min_action, max_action + + # Create Uniform Bins + Compute Bin Centers + self.bins = self.get_bins(min_action, max_action, self.n_bins, use_norm_bins) + self.bin_centers = (self.bins[:-1] + self.bins[1:]) / 2.0 + + # [Contract] Set "action_token_begin_idx" based on `self.tokenizer.vocab_size - (self.n_bins + 1)` + # =>> Assumes we're always overwriting the final `n_bins` tokens of the vocabulary! + self.action_token_begin_idx: int = int(self.tokenizer.vocab_size - (self.n_bins + 1)) + + def __call__(self, action: np.ndarray) -> Union[str, List[str]]: + """Clip & bin actions to *the last `n_bins` tokens* of the vocabulary (e.g., tokenizer.vocab[-256:]).""" + action = np.clip(action, a_min=float(self.min_action), a_max=float(self.max_action)) + discretized_action = np.digitize(action, self.bins) + + # Handle single element vs. batch + if len(discretized_action.shape) == 1: + return self.tokenizer.decode(list(self.tokenizer.vocab_size - discretized_action)) + else: + return self.tokenizer.batch_decode((self.tokenizer.vocab_size - discretized_action).tolist()) + + def decode_token_ids_to_actions(self, action_token_ids: np.ndarray) -> np.ndarray: + """ + Returns continuous actions for discrete action token IDs. + + NOTE =>> Because of the way the actions are discretized w.r.t. the bins (and not the bin centers), the + digitization returns bin indices between [1, # bins], inclusive, when there are actually only + (# bins - 1) bin intervals. + + Therefore, if the digitization returns the last possible index, we map this to the last bin interval. + + EXAMPLE =>> Let's say self._bins has 256 values. Then self._bin_centers has 255 values. Digitization returns + indices between [1, 256]. We subtract 1 from all indices so that they are between [0, 255]. There + is still one index (i==255) that would cause an out-of-bounds error if used to index into + self._bin_centers. Therefore, if i==255, we subtract 1 from it so that it just becomes the index of + the last bin center. We implement this simply via clipping between [0, 255 - 1]. + """ + discretized_actions = self.tokenizer.vocab_size - action_token_ids + discretized_actions = np.clip(discretized_actions - 1, a_min=0, a_max=self.bin_centers.shape[0] - 1) + + return self.bin_centers[discretized_actions] + + @property + def vocab_size(self) -> int: + return self.n_bins + + def get_bins(self, min_action, max_action, n_bins, use_norm_bins=False): + if use_norm_bins: + a = np.linspace( 0.5, 1.0, 15, endpoint=True) + b = np.linspace( 0.25, 0.5, 40, endpoint=False) + c = np.linspace( 0.1, 0.25, 42, endpoint=False) + d = np.linspace(-0.1, 0.1, 62, endpoint=False) + e = np.linspace(-0.25, -0.1, 42, endpoint=False) + f = np.linspace(-0.5, -0.25, 40, endpoint=False) + g = np.linspace(-1, -0.5, 15, endpoint=False) + return np.concatenate((g, f, e, d, c, b, a), axis=0) + else: + return np.linspace(min_action, max_action, n_bins) + + +def encode_actions_real(sentence, action_tokenizer, statistics=None): + if statistics is not None: + assert Path(statistics).exists(), "Statistical file not found." + with open(statistics, "r") as file_: + config = yaml.safe_load(file_) + act_std = np.array(config["act_min_bound"]) + act_mean = np.array(config["act_max_bound"]) + + act_mean = np.tile(act_mean, 5) + act_std = np.tile(act_std, 5) + + actions = sentence.split(" ") + actions = [float(action) for action in actions] + actions = (np.array(actions) - act_mean) / (act_std + 1e-5) + + # 3 sigma clipping + actions = actions / 3 + actions = np.clip(actions, a_min=-1, a_max=1) + actions_lang = action_tokenizer(actions) + return actions_lang + + +def encode_actions(sentence, action_tokenizer, statistics=None): + if statistics is not None: + print("use statistics") + # use the absolute action + assert Path(statistics).exists(), "Statistical file not found." + with open(statistics, "r") as file_: + config = yaml.safe_load(file_) + act_min_bound = np.array(config["act_min_bound"]) + act_max_bound = np.array(config["act_max_bound"]) + act_min_bound = np.tile(act_min_bound, 5) + act_max_bound = np.tile(act_max_bound, 5) + + actions = sentence.split(" ") + actions = [float(action) for action in actions] + actions = (np.array(actions) - act_min_bound) / (act_max_bound - act_min_bound) + actions = actions * 2 - 1 + actions_lang = action_tokenizer(actions) + else: + # use the relative action + # print("not use statistics") + actions = sentence.split(" ") + actions = [float(action) for action in actions] + actions_lang = action_tokenizer(actions) + + return actions_lang +def encode_actions_forpipper(sentence, action_tokenizer, statistics=None): + assert statistics is not None, "Statistics file path is required." + assert Path(statistics).exists(), "Statistical file not found." + + # 读取 JSON + with open(statistics, "r") as f: + config = json.load(f) + + # 从 JSON 中提取动作最小值 / 最大值 + act_min_bound = np.array(config["action"]["min"]) + act_max_bound = np.array(config["action"]["max"]) + + # 支持 Tensor 或字符串输入 + if isinstance(sentence, torch.Tensor): + actions = sentence.cpu().numpy() + elif isinstance(sentence, list): + actions = np.array(sentence) + elif isinstance(sentence, str): + actions = np.array([float(a) for a in sentence.strip().split()]) + else: + raise TypeError("Unsupported type for `sentence`") + + action_dim = len(act_min_bound) + + # 如果 actions 是多个动作拼接(如两帧),自动扩展 + if len(actions) != action_dim: + assert len(actions) % action_dim == 0, f"动作长度 {len(actions)} 不是基础维度 {action_dim} 的整数倍" + repeat_factor = len(actions) // action_dim + act_min_bound = np.tile(act_min_bound, repeat_factor) + act_max_bound = np.tile(act_max_bound, repeat_factor) + + # Min-max 归一化到 [-1, 1] + normed_actions = (actions - act_min_bound) / (act_max_bound - act_min_bound + 1e-5) + normed_actions = normed_actions * 2 - 1 + + # Tokenizer 处理 + actions_lang = action_tokenizer(normed_actions) + return actions_lang +def decode_actions_forpipper(normed_actions, statistics=None): + assert statistics is not None, "Statistics file path is required." + assert Path(statistics).exists(), "Statistical file not found." + + # 读取 JSON 文件,提取动作范围 + with open(statistics, "r") as f: + config = json.load(f) + + act_min_bound = np.array(config["action"]["min"]) + act_max_bound = np.array(config["action"]["max"]) + + # 如果传入的是 Tensor / list / str,都统一转换为 numpy + if isinstance(normed_actions, torch.Tensor): + normed_actions = normed_actions.cpu().numpy() + elif isinstance(normed_actions, list): + normed_actions = np.array(normed_actions) + elif isinstance(normed_actions, str): + normed_actions = np.array([float(a) for a in normed_actions.strip().split()]) + + action_dim = len(act_min_bound) + + # 处理重复动作帧情况(如拼接了多帧) + if len(normed_actions) != action_dim: + assert len(normed_actions) % action_dim == 0, \ + f"动作长度 {len(normed_actions)} 不是基础维度 {action_dim} 的整数倍" + repeat_factor = len(normed_actions) // action_dim + act_min_bound = np.tile(act_min_bound, repeat_factor) + act_max_bound = np.tile(act_max_bound, repeat_factor) + + # 反归一化:[-1, 1] → [min, max] + normed_actions = (normed_actions + 1) / 2 + actions = normed_actions * (act_max_bound - act_min_bound + 1e-5) + act_min_bound + + return actions +def denormalize_actions(actions: np.float32, statistics=None): + assert Path(statistics).exists(), "Statistical file not found." + with open(statistics, "r") as file_: + config = yaml.safe_load(file_) + act_min_bound = np.array(config["act_min_bound"]) + act_max_bound = np.array(config["act_max_bound"]) + + times = len(actions) // len(act_min_bound) + act_min_bound = np.tile(act_min_bound, times) + act_max_bound = np.tile(act_max_bound, times) + actions = (actions + 1) / 2 + actions = actions * (act_max_bound - act_min_bound) + act_min_bound + return actions + + +def denormalize_actions_real(actions: np.float32, statistics=None): + assert Path(statistics).exists(), "Statistical file not found." + with open(statistics, "r") as file_: + config = yaml.safe_load(file_) + act_std = np.array(config["act_min_bound"]) + act_mean = np.array(config["act_max_bound"]) + + act_mean = np.tile(act_mean, 5) + act_std = np.tile(act_std, 5) + actions = actions * 3 + actions = actions * act_std + act_mean + return actions + + +def encode_robot_obs(sentence, action_tokenizer, statistics=None): + assert Path(statistics).exists(), "Statistical file not found." + with open(statistics, "r") as file_: + config = yaml.safe_load(file_) + + robot_obs_mean = np.array(config["robot_obs"][0]["mean"]) + robot_obs_std = np.array(config["robot_obs"][0]["std"]) + + robot_obs = sentence.split(" ") + assert len(robot_obs) == 15, "The observation of the robot is mismatch." + robot_obs = [float(item) for item in robot_obs] + + # According to 3 sigma principle, value is in the range of (-3, 3). + robot_obs = (np.array(robot_obs) - robot_obs_mean) / (robot_obs_std + 1e-5) + + # Convert value to the range of (-1, 1). + robot_obs = robot_obs / 3.0 + + robot_obs_lang = action_tokenizer(robot_obs) + return robot_obs_lang + +def encode_robot_obs_forpipper(sentence, action_tokenizer, statistics=None): + assert Path(statistics).exists(), "Statistical file not found." + + with open(statistics, "r") as f: + stats = json.load(f) + + # 读取 observation.state 部分的 mean 和 std(注意:不是 robot_obs,而是 observation.state) + obs_mean = np.array(stats["observation.state"]["mean"]) + obs_std = np.array(stats["observation.state"]["std"]) + + # ✅ 支持 tensor / list / str + if isinstance(sentence, torch.Tensor): + robot_obs = sentence.cpu().numpy().tolist() + elif isinstance(sentence, list): + robot_obs = sentence + elif isinstance(sentence, str): + robot_obs = [float(x) for x in sentence.strip().split()] + else: + raise TypeError("Unsupported input type for `sentence`") + + assert len(robot_obs) == len(obs_mean), f"Expected {len(obs_mean)} obs values, got {len(robot_obs)}" + + # 归一化:根据 3σ 原则缩放至 [-1, 1] + robot_obs = (np.array(robot_obs) - obs_mean) / (obs_std + 1e-5) + robot_obs = robot_obs / 3.0 # 缩放至 [-1, 1] + + # 使用 action_tokenizer 进行离散化(或文本化) + robot_obs_lang = action_tokenizer(robot_obs) + return robot_obs_lang + diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/LLaVA-VLA/llava/action_tokenizer_calvin.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/LLaVA-VLA/llava/action_tokenizer_calvin.py new file mode 100644 index 00000000..89ff4890 --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/LLaVA-VLA/llava/action_tokenizer_calvin.py @@ -0,0 +1,187 @@ +""" +action_tokenizer.py + +Extension class; wraps base LLM/VLM tokenizer with logic to discretize and tokenize continuous robot actions. +""" + +from typing import List, Union + +import numpy as np +from transformers import PreTrainedTokenizerBase +from pathlib import Path +import yaml + + +class ActionTokenizer: + def __init__( + self, tokenizer: PreTrainedTokenizerBase, bins: int = 256, min_action: int = -1, max_action: int = 1 + ) -> None: + """ + Discretizes continuous robot actions into N bins per dimension and maps to the least used tokens. + + NOTE =>> by default, assumes a BPE-style tokenizer akin to the LlamaTokenizer, where *the least used tokens* + appear at the end of the vocabulary! + + :param tokenizer: Base LLM/VLM tokenizer to extend. + :param bins: Number of bins for each continuous value; we'll adopt a uniform binning strategy. + :param min_action: Minimum action value (for clipping, setting lower bound on bin interval). + :param max_action: Maximum action value (for clipping, setting upper bound on bin interval). + """ + self.tokenizer, self.n_bins, self.min_action, self.max_action = tokenizer, bins, min_action, max_action + + # Create Uniform Bins + Compute Bin Centers + self.bins = self.get_bins(min_action, max_action, self.n_bins, use_norm_bins=False) + self.bin_centers = (self.bins[:-1] + self.bins[1:]) / 2.0 + + # [Contract] Set "action_token_begin_idx" based on `self.tokenizer.vocab_size - (self.n_bins + 1)` + # =>> Assumes we're always overwriting the final `n_bins` tokens of the vocabulary! + self.action_token_begin_idx: int = int(self.tokenizer.vocab_size - (self.n_bins + 1)) + + def __call__(self, action: np.ndarray) -> Union[str, List[str]]: + """Clip & bin actions to *the last `n_bins` tokens* of the vocabulary (e.g., tokenizer.vocab[-256:]).""" + action = np.clip(action, a_min=float(self.min_action), a_max=float(self.max_action)) + discretized_action = np.digitize(action, self.bins) + + # Handle single element vs. batch + if len(discretized_action.shape) == 1: + return self.tokenizer.decode(list(self.tokenizer.vocab_size - discretized_action)) + else: + return self.tokenizer.batch_decode((self.tokenizer.vocab_size - discretized_action).tolist()) + + def decode_token_ids_to_actions(self, action_token_ids: np.ndarray) -> np.ndarray: + """ + Returns continuous actions for discrete action token IDs. + + NOTE =>> Because of the way the actions are discretized w.r.t. the bins (and not the bin centers), the + digitization returns bin indices between [1, # bins], inclusive, when there are actually only + (# bins - 1) bin intervals. + + Therefore, if the digitization returns the last possible index, we map this to the last bin interval. + + EXAMPLE =>> Let's say self._bins has 256 values. Then self._bin_centers has 255 values. Digitization returns + indices between [1, 256]. We subtract 1 from all indices so that they are between [0, 255]. There + is still one index (i==255) that would cause an out-of-bounds error if used to index into + self._bin_centers. Therefore, if i==255, we subtract 1 from it so that it just becomes the index of + the last bin center. We implement this simply via clipping between [0, 255 - 1]. + """ + discretized_actions = self.tokenizer.vocab_size - action_token_ids + discretized_actions = np.clip(discretized_actions - 1, a_min=0, a_max=self.bin_centers.shape[0] - 1) + + return self.bin_centers[discretized_actions] + + @property + def vocab_size(self) -> int: + return self.n_bins + + def get_bins(self, min_action, max_action, n_bins, use_norm_bins=False): + if use_norm_bins: + a = np.linspace( 0.5, 1.0, 15, endpoint=True) + b = np.linspace( 0.25, 0.5, 40, endpoint=False) + c = np.linspace( 0.1, 0.25, 42, endpoint=False) + d = np.linspace(-0.1, 0.1, 62, endpoint=False) + e = np.linspace(-0.25, -0.1, 42, endpoint=False) + f = np.linspace(-0.5, -0.25, 40, endpoint=False) + g = np.linspace(-1, -0.5, 15, endpoint=False) + return np.concatenate((g, f, e, d, c, b, a), axis=0) + else: + return np.linspace(min_action, max_action, n_bins) + + +def encode_actions_real(sentence, action_tokenizer, statistics=None): + if statistics is not None: + assert Path(statistics).exists(), "Statistical file not found." + with open(statistics, "r") as file_: + config = yaml.safe_load(file_) + act_std = np.array(config["act_min_bound"]) + act_mean = np.array(config["act_max_bound"]) + + act_mean = np.tile(act_mean, 5) + act_std = np.tile(act_std, 5) + + actions = sentence.split(" ") + actions = [float(action) for action in actions] + actions = (np.array(actions) - act_mean) / act_std + + # 3 sigma clipping + actions = actions / 3 + actions = np.clip(actions, a_min=-1, a_max=1) + actions_lang = action_tokenizer(actions) + return actions_lang + + +def encode_actions(sentence, action_tokenizer, statistics=None): + if statistics is not None: + # use the absolute action + assert Path(statistics).exists(), "Statistical file not found." + with open(statistics, "r") as file_: + config = yaml.safe_load(file_) + act_min_bound = np.array(config["act_min_bound"]) + act_max_bound = np.array(config["act_max_bound"]) + act_min_bound = np.tile(act_min_bound, 5) + act_max_bound = np.tile(act_max_bound, 5) + + actions = sentence.split(" ") + actions = [float(action) for action in actions] + actions = (np.array(actions) - act_min_bound) / (act_max_bound - act_min_bound) + actions = actions * 2 - 1 + actions_lang = action_tokenizer(actions) + else: + # use the relative action + actions = sentence.split(" ") + actions = [float(action) for action in actions] + actions_lang = action_tokenizer(actions) + return actions_lang + + +def denormalize_actions(actions: np.float32, statistics=None): + assert Path(statistics).exists(), "Statistical file not found." + with open(statistics, "r") as file_: + config = yaml.safe_load(file_) + act_min_bound = np.array(config["act_min_bound"]) + act_max_bound = np.array(config["act_max_bound"]) + + times = len(actions) // len(act_min_bound) + act_min_bound = np.tile(act_min_bound, times) + act_max_bound = np.tile(act_max_bound, times) + actions = (actions + 1) / 2 + actions = actions * (act_max_bound - act_min_bound) + act_min_bound + return actions + + +def denormalize_actions_real(actions: np.float32, statistics=None): + assert Path(statistics).exists(), "Statistical file not found." + with open(statistics, "r") as file_: + config = yaml.safe_load(file_) + act_std = np.array(config["act_min_bound"]) + act_mean = np.array(config["act_max_bound"]) + + act_mean = np.tile(act_mean, 5) + act_std = np.tile(act_std, 5) + actions = actions * 3 + actions = actions * act_std + act_mean + return actions + + +def encode_robot_obs(sentence, action_tokenizer, statistics=None): + assert Path(statistics).exists(), "Statistical file not found." + with open(statistics, "r") as file_: + config = yaml.safe_load(file_) + + robot_obs_mean = np.array(config["robot_obs"][0]["mean"]) + robot_obs_std = np.array(config["robot_obs"][0]["std"]) + + robot_obs = sentence.split(" ") + assert len(robot_obs) == 15 or len(robot_obs) == 75, "The observation of the robot is mismatch." + robot_obs = [float(item) for item in robot_obs] + if len(robot_obs) == 75: + robot_obs_mean = np.tile(robot_obs_mean, 5) + robot_obs_std = np.tile(robot_obs_std, 5) + + # According to 3 sigma principle, value is in the range of (-3, 3). + robot_obs = (np.array(robot_obs) - robot_obs_mean) / (robot_obs_std + 0.000001) + + # Convert value to the range of (-1, 1). + robot_obs = robot_obs / 3.0 + + robot_obs_lang = action_tokenizer(robot_obs) + return robot_obs_lang diff --git a/operator-images/builtin/holobrain-vendor/RoboTwin/policy/LLaVA-VLA/llava/constants.py b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/LLaVA-VLA/llava/constants.py new file mode 100644 index 00000000..6ba1d48a --- /dev/null +++ b/operator-images/builtin/holobrain-vendor/RoboTwin/policy/LLaVA-VLA/llava/constants.py @@ -0,0 +1,17 @@ +CONTROLLER_HEART_BEAT_EXPIRATION = 30 +WORKER_HEART_BEAT_INTERVAL = 15 + +LOGDIR = "." + +# Model Constants +IGNORE_INDEX = -100 +IMAGE_TOKEN_INDEX = -200 +DEFAULT_IMAGE_TOKEN = "" +DEFAULT_IMAGE_PATCH_TOKEN = "" +DEFAULT_IM_START_TOKEN = "" +DEFAULT_IM_END_TOKEN = "" +IMAGE_PLACEHOLDER = "" +DEFAULT_AUDIO_TOKEN = "