droboticflow/backend/operators/builtin/structured_model_operator.py
2026-07-02 12:54:48 +08:00

874 lines
35 KiB
Python

from __future__ import annotations
import csv
import contextlib
import datetime as dt
import hashlib
import json
import math
import os
import pathlib
import shutil
import subprocess
import zipfile
from typing import Any, Callable
HelperMap = dict[str, Callable[..., Any]]
def _text(params: dict[str, str], key: str, default: str = "") -> str:
return str(params.get(key, default) or default).strip()
def _int(params: dict[str, str], key: str, default: int) -> int:
return _int_at_least(params, key, default, 1)
def _int_at_least(params: dict[str, str], key: str, default: int, minimum: int) -> int:
value = _text(params, key, str(default))
try:
parsed = int(value)
except ValueError as exc:
raise RuntimeError(f"{key} must be an integer") from exc
if parsed < minimum:
raise RuntimeError(f"{key} must be >= {minimum}")
return parsed
def _float(params: dict[str, str], key: str, default: float) -> float:
value = _text(params, key, str(default))
try:
parsed = float(value)
except ValueError as exc:
raise RuntimeError(f"{key} must be a number") from exc
if not math.isfinite(parsed):
raise RuntimeError(f"{key} must be finite")
return parsed
def _bool(params: dict[str, str], key: str, default: bool) -> bool:
value = _text(params, key, "true" if default else "false").lower()
if value in {"1", "true", "yes", "on"}:
return True
if value in {"0", "false", "no", "off"}:
return False
raise RuntimeError(f"{key} must be a boolean")
def _json_list(params: dict[str, str], key: str) -> list[Any]:
value = _text(params, key)
if not value:
return []
payload = json.loads(value)
if not isinstance(payload, list):
raise RuntimeError(f"{key} must be a JSON array")
return payload
def _dataset_path(params: dict[str, str], inputs: list[dict[str, Any]]) -> pathlib.Path:
explicit = _text(params, "DATASET_PATH") or _text(params, "TRAIN_DATA_PATH") or _text(params, "SOURCE_PATH")
if explicit:
path = pathlib.Path(explicit)
if not path.exists():
raise RuntimeError(f"dataset path does not exist: {path}")
return path
for artifact in inputs:
path = pathlib.Path(str(artifact.get("path") or artifact.get("uri") or ""))
if path.exists():
return path
raise RuntimeError("structured model operator requires DATASET_PATH/TRAIN_DATA_PATH/SOURCE_PATH or an input artifact path")
def _prepare_transformers_cache(params: dict[str, str], output_dir: pathlib.Path) -> pathlib.Path:
explicit_cache_dir = _text(params, "HF_CACHE_DIR") or _text(params, "TRANSFORMERS_CACHE_DIR")
if explicit_cache_dir:
cache_dir = pathlib.Path(explicit_cache_dir)
else:
cache_key = hashlib.sha256(str(output_dir).encode("utf-8")).hexdigest()[:16]
cache_dir = pathlib.Path("/tmp") / "droboticflow-hf-cache" / cache_key
cache_dir.mkdir(parents=True, exist_ok=True)
if not os.access(cache_dir, os.W_OK):
raise RuntimeError(f"transformers cache directory is not writable: {cache_dir}")
os.environ["HF_HUB_DISABLE_SYMLINKS_WARNING"] = "1"
os.environ.setdefault("HF_HUB_DISABLE_XET", "1")
os.environ["HF_HOME"] = str(cache_dir)
os.environ["HUGGINGFACE_HUB_CACHE"] = str(cache_dir / "hub")
os.environ["TRANSFORMERS_CACHE"] = str(cache_dir / "transformers")
return cache_dir
def _find_first_file(root: pathlib.Path, suffixes: tuple[str, ...]) -> pathlib.Path | None:
if root.is_file() and root.suffix.lower() in suffixes:
return root
if not root.exists():
return None
candidates = [path for path in root.rglob("*") if path.is_file() and path.suffix.lower() in suffixes]
if not candidates:
return None
priority_names = ("best.pt", "latest.ckpt", "last.pt")
for name in priority_names:
for candidate in candidates:
if candidate.name == name:
return candidate
candidates.sort(key=lambda path: path.stat().st_mtime, reverse=True)
return candidates[0]
def _zip_directory(source_dir: pathlib.Path, archive_path: pathlib.Path) -> pathlib.Path:
if not source_dir.exists() or not source_dir.is_dir():
raise RuntimeError(f"model directory does not exist: {source_dir}")
with zipfile.ZipFile(archive_path, "w", compression=zipfile.ZIP_DEFLATED) as archive:
for item in sorted(source_dir.rglob("*")):
if item.is_file():
archive.write(item, item.relative_to(source_dir.parent))
return archive_path
def _run_framework_command(
command: list[str],
*,
cwd: pathlib.Path | None,
output_dir: pathlib.Path,
framework: str,
metadata: dict[str, Any],
env: dict[str, str] | None = None,
) -> dict[str, Any]:
output_dir.mkdir(parents=True, exist_ok=True)
started = dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z")
result = subprocess.run(
command,
cwd=str(cwd) if cwd else None,
env=env,
text=True,
capture_output=True,
check=False,
)
(output_dir / "framework-stdout.log").write_text(result.stdout or "", encoding="utf-8")
(output_dir / "framework-stderr.log").write_text(result.stderr or "", encoding="utf-8")
payload = {
"framework": framework,
"command": command,
"cwd": str(cwd) if cwd else "",
"returnCode": result.returncode,
"startedAt": started,
**metadata,
}
(output_dir / "framework-command.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
if result.returncode != 0:
raise RuntimeError(f"{framework} training command failed with exit code {result.returncode}; see framework-stderr.log")
return payload
def _framework_artifacts(
helpers: HelperMap,
run_id: str,
node_id: str,
output_dir: pathlib.Path,
metrics: dict[str, Any],
model_path: pathlib.Path,
) -> list[dict[str, Any]]:
if not model_path.exists():
raise RuntimeError(f"expected framework model artifact was not produced: {model_path}")
return [
helpers["write_json_artifact"](run_id, node_id, output_dir, "metrics.json", metrics),
helpers["build_artifact"](run_id, node_id, output_dir / "framework-command.json", "json"),
helpers["build_artifact"](run_id, node_id, output_dir / "framework-stdout.log", "log"),
helpers["build_artifact"](run_id, node_id, output_dir / "framework-stderr.log", "log"),
helpers["build_artifact"](run_id, node_id, model_path, "model"),
]
def _load_json_records(path: pathlib.Path) -> list[dict[str, Any]]:
if path.is_dir():
candidates = [
path / "samples.jsonl",
path / "samples.json",
path / "dataset.jsonl",
path / "dataset.json",
path / "manifest.jsonl",
path / "manifest.json",
]
for candidate in candidates:
if candidate.exists():
return _load_json_records(candidate)
raise RuntimeError(f"dataset directory does not contain a supported JSON/JSONL manifest: {path}")
if path.suffix.lower() == ".jsonl":
records: list[dict[str, Any]] = []
for line in path.read_text(encoding="utf-8").splitlines():
if line.strip():
payload = json.loads(line)
if isinstance(payload, dict):
records.append(payload)
return records
if path.suffix.lower() == ".csv":
with path.open(newline="", encoding="utf-8") as handle:
return [dict(row) for row in csv.DictReader(handle)]
payload = json.loads(path.read_text(encoding="utf-8"))
if isinstance(payload, dict):
for key in ("samples", "items", "records", "data"):
value = payload.get(key)
if isinstance(value, list):
return [item for item in value if isinstance(item, dict)]
return [payload]
if isinstance(payload, list):
return [item for item in payload if isinstance(item, dict)]
raise RuntimeError(f"dataset must be a JSON object/list, JSONL, or CSV: {path}")
def _vector(value: Any) -> list[float]:
if isinstance(value, str):
try:
value = json.loads(value)
except json.JSONDecodeError:
value = [float(part) for part in value.replace(";", ",").split(",") if part.strip()]
if isinstance(value, (int, float)):
value = [value]
if not isinstance(value, list) or not value:
raise RuntimeError("state/action fields must be numeric arrays")
vector = [float(item) for item in value]
if not all(math.isfinite(item) for item in vector):
raise RuntimeError("state/action vectors must contain finite values")
return vector
def _field_value(item: dict[str, Any], field: str) -> Any:
value: Any = item
for part in field.split("."):
if isinstance(value, dict):
value = value.get(part)
else:
return None
return value
def run_policy_training(
run_id: str,
node_id: str,
output_dir: Any,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
helpers: HelperMap,
*,
operator_name: str,
model_family: str,
) -> list[dict[str, Any]]:
try:
import torch
from torch import nn
from torch.utils.data import DataLoader, TensorDataset
except Exception as exc: # pragma: no cover - exercised in container images
raise RuntimeError("PyTorch is required in the operator image for structured policy training") from exc
params = helpers["hyperparams"](node_data)
dataset_path = _dataset_path(params, inputs)
records = _load_json_records(dataset_path)
if len(records) < 2:
raise RuntimeError("policy training requires at least 2 real samples")
state_field = _text(params, "STATE_FIELD", "state")
action_field = _text(params, "ACTION_FIELD", "action")
states = [_vector(_field_value(item, state_field)) for item in records]
actions = [_vector(_field_value(item, action_field)) for item in records]
state_dim = len(states[0])
action_dim = len(actions[0])
if any(len(item) != state_dim for item in states) or any(len(item) != action_dim for item in actions):
raise RuntimeError("all state/action vectors must have consistent dimensions")
epochs = _int(params, "EPOCHS", 5)
batch_size = _int(params, "BATCH_SIZE", 8)
learning_rate = _float(params, "LEARNING_RATE", 0.001)
hidden_dim = _int(params, "HIDDEN_DIM", 128)
chunk_size = _int(params, "CHUNK_SIZE", 1)
x = torch.tensor(states, dtype=torch.float32)
y = torch.tensor(actions, dtype=torch.float32)
loader = DataLoader(TensorDataset(x, y), batch_size=min(batch_size, len(records)), shuffle=True)
model = nn.Sequential(nn.Linear(state_dim, hidden_dim), nn.ReLU(), nn.Linear(hidden_dim, action_dim))
optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)
loss_fn = nn.MSELoss()
losses: list[float] = []
for _ in range(epochs):
epoch_loss = 0.0
for batch_x, batch_y in loader:
optimizer.zero_grad(set_to_none=True)
loss = loss_fn(model(batch_x), batch_y)
loss.backward()
optimizer.step()
epoch_loss += float(loss.detach().cpu())
losses.append(epoch_loss / max(len(loader), 1))
output_path = pathlib.Path(output_dir)
checkpoint_path = output_path / "checkpoint.pt"
torch.save(
{
"operator": operator_name,
"modelFamily": model_family,
"stateField": state_field,
"actionField": action_field,
"stateDim": state_dim,
"actionDim": action_dim,
"chunkSize": chunk_size,
"modelStateDict": model.state_dict(),
},
checkpoint_path,
)
metrics = {
"operator": operator_name,
"modelFamily": model_family,
"trainingBackend": "torch_mlp",
"datasetPath": str(dataset_path),
"sampleCount": len(records),
"stateDim": state_dim,
"actionDim": action_dim,
"epochs": epochs,
"batchSize": batch_size,
"learningRate": learning_rate,
"chunkSize": chunk_size,
"finalLoss": losses[-1],
"losses": losses,
}
return [
helpers["write_json_artifact"](run_id, node_id, output_path, "metrics.json", metrics),
helpers["build_artifact"](run_id, node_id, checkpoint_path, "model"),
]
def run_lerobot_act_training(
run_id: str,
node_id: str,
output_dir: Any,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
helpers: HelperMap,
) -> list[dict[str, Any]]:
params = helpers["hyperparams"](node_data)
backend = _text(params, "TRAINING_BACKEND", "lerobot").lower()
if backend == "torch_mlp":
return run_policy_training(
run_id,
node_id,
output_dir,
node_data,
inputs,
helpers,
operator_name="ACT 策略训练",
model_family="ACT",
)
if backend != "lerobot":
raise RuntimeError("ACT TRAINING_BACKEND must be lerobot or torch_mlp")
dataset_repo_id = _text(params, "DATASET_REPO_ID")
dataset_root = _text(params, "DATASET_PATH") or _text(params, "TRAIN_DATA_PATH")
if not dataset_repo_id and not dataset_root:
raise RuntimeError("ACT LeRobot training requires DATASET_REPO_ID or DATASET_PATH")
if dataset_root and not pathlib.Path(dataset_root).exists():
raise RuntimeError(f"ACT DATASET_PATH does not exist: {dataset_root}")
output_path = pathlib.Path(output_dir)
train_dir = output_path / "lerobot-act"
command = [
_text(params, "LEROBOT_TRAIN_BIN", "lerobot-train"),
f"--policy.type={_text(params, 'POLICY_TYPE', 'act')}",
f"--output_dir={train_dir}",
f"--job_name={_text(params, 'JOB_NAME', f'droboticflow-act-{run_id[:8]}')}",
f"--policy.device={_text(params, 'DEVICE', 'cuda')}",
f"--wandb.enable={str(_bool(params, 'WANDB_ENABLE', False)).lower()}",
]
if dataset_repo_id:
command.append(f"--dataset.repo_id={dataset_repo_id}")
if dataset_root:
command.append(f"--dataset.root={dataset_root}")
policy_repo_id = _text(params, "POLICY_REPO_ID")
if policy_repo_id:
command.append(f"--policy.repo_id={policy_repo_id}")
if _text(params, "MAX_STEPS"):
command.append(f"--steps={_int(params, 'MAX_STEPS', 1000)}")
if _text(params, "BATCH_SIZE"):
command.append(f"--batch_size={_int(params, 'BATCH_SIZE', 8)}")
if _text(params, "LEARNING_RATE"):
command.append(f"--optimizer.lr={_float(params, 'LEARNING_RATE', 0.0001)}")
if _text(params, "CHUNK_SIZE"):
command.append(f"--policy.chunk_size={_int(params, 'CHUNK_SIZE', 100)}")
for override in _json_list(params, "LEROBOT_OVERRIDES_JSON"):
if not isinstance(override, str) or not override.strip() or override.strip().startswith("-"):
raise RuntimeError("LEROBOT_OVERRIDES_JSON entries must be non-empty Hydra override strings")
command.append(override.strip())
_run_framework_command(
command,
cwd=None,
output_dir=output_path,
framework="LeRobot ACT",
metadata={
"datasetRepoId": dataset_repo_id,
"datasetRoot": dataset_root,
"trainingBackend": backend,
},
env={**os.environ, "HF_HUB_DISABLE_XET": os.environ.get("HF_HUB_DISABLE_XET", "1")},
)
checkpoint = _find_first_file(train_dir, (".safetensors", ".pt", ".pth", ".bin"))
if checkpoint is None:
raise RuntimeError("LeRobot ACT training completed but no checkpoint file was found")
copied_checkpoint = output_path / checkpoint.name
if checkpoint != copied_checkpoint:
shutil.copy2(checkpoint, copied_checkpoint)
metrics = {
"operator": "ACT 策略训练",
"modelFamily": "LeRobot ACT",
"trainingBackend": backend,
"datasetRepoId": dataset_repo_id,
"datasetRoot": dataset_root,
"outputDir": str(train_dir),
"checkpoint": str(copied_checkpoint),
"device": _text(params, "DEVICE", "cuda"),
}
return _framework_artifacts(helpers, run_id, node_id, output_path, metrics, copied_checkpoint)
def run_diffusion_policy_training(
run_id: str,
node_id: str,
output_dir: Any,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
helpers: HelperMap,
) -> list[dict[str, Any]]:
params = helpers["hyperparams"](node_data)
backend = _text(params, "TRAINING_BACKEND", "diffusion_policy").lower()
if backend == "torch_mlp":
return run_policy_training(
run_id,
node_id,
output_dir,
node_data,
inputs,
helpers,
operator_name="Diffusion Policy 训练",
model_family="Diffusion Policy",
)
if backend != "diffusion_policy":
raise RuntimeError("Diffusion Policy TRAINING_BACKEND must be diffusion_policy or torch_mlp")
repo_root = pathlib.Path(_text(params, "DIFFUSION_POLICY_ROOT") or os.environ.get("DIFFUSION_POLICY_ROOT", "/opt/diffusion_policy"))
train_py = repo_root / "train.py"
if not train_py.exists():
raise RuntimeError(f"Diffusion Policy train.py does not exist: {train_py}")
dataset_path = _dataset_path(params, inputs)
output_path = pathlib.Path(output_dir)
train_dir = output_path / "diffusion-policy"
config_name = _text(params, "CONFIG_NAME", "image_pusht_diffusion_policy_cnn")
command = [
"python",
str(train_py),
"--config-dir",
_text(params, "CONFIG_DIR", "."),
f"--config-name={config_name}",
f"hydra.run.dir={train_dir}",
f"training.device={_text(params, 'DEVICE', 'cuda:0')}",
]
task_name = _text(params, "TASK_NAME")
if task_name:
command.append(f"task={task_name}")
command.extend(
[
f"task.dataset.zarr_path={dataset_path}",
f"training.num_epochs={_int(params, 'EPOCHS', 1)}",
f"dataloader.batch_size={_int(params, 'BATCH_SIZE', 8)}",
f"optimizer.lr={_float(params, 'LEARNING_RATE', 0.0001)}",
]
)
if _text(params, "HORIZON"):
command.append(f"horizon={_int(params, 'HORIZON', 16)}")
if _text(params, "DIFFUSION_STEPS"):
command.append(f"policy.num_inference_steps={_int(params, 'DIFFUSION_STEPS', 32)}")
for override in _json_list(params, "HYDRA_OVERRIDES_JSON"):
if not isinstance(override, str) or not override.strip() or override.strip().startswith("-"):
raise RuntimeError("HYDRA_OVERRIDES_JSON entries must be non-empty Hydra override strings")
command.append(override.strip())
_run_framework_command(
command,
cwd=repo_root,
output_dir=output_path,
framework="real-stanford Diffusion Policy",
metadata={
"datasetPath": str(dataset_path),
"trainingBackend": backend,
"configName": config_name,
"taskName": task_name,
},
env={**os.environ, "WANDB_MODE": os.environ.get("WANDB_MODE", "offline")},
)
checkpoint = _find_first_file(train_dir, (".ckpt", ".pt", ".pth"))
if checkpoint is None:
raise RuntimeError("Diffusion Policy training completed but no checkpoint file was found")
copied_checkpoint = output_path / checkpoint.name
if checkpoint != copied_checkpoint:
shutil.copy2(checkpoint, copied_checkpoint)
metrics = {
"operator": "Diffusion Policy 训练",
"modelFamily": "real-stanford Diffusion Policy",
"trainingBackend": backend,
"datasetPath": str(dataset_path),
"configName": config_name,
"taskName": task_name,
"outputDir": str(train_dir),
"checkpoint": str(copied_checkpoint),
"device": _text(params, "DEVICE", "cuda:0"),
}
return _framework_artifacts(helpers, run_id, node_id, output_path, metrics, copied_checkpoint)
def run_yolo_training(
run_id: str,
node_id: str,
output_dir: Any,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
helpers: HelperMap,
) -> list[dict[str, Any]]:
try:
from ultralytics import YOLO
except Exception as exc: # pragma: no cover - exercised in container images
raise RuntimeError("Ultralytics is required in the YOLO operator image") from exc
params = helpers["hyperparams"](node_data)
data_yaml = pathlib.Path(_text(params, "DATA_YAML") or _text(params, "DATASET_YAML") or str(_dataset_path(params, inputs)))
if not data_yaml.exists():
raise RuntimeError(f"YOLO data.yaml does not exist: {data_yaml}")
model_name = _text(params, "MODEL_SIZE", "yolo11n.pt")
epochs = _int(params, "EPOCHS", 10)
image_size = _int(params, "IMAGE_SIZE", 640)
batch_size = _int(params, "BATCH_SIZE", 8)
workers = _int_at_least(params, "WORKERS", 0, 0)
output_path = pathlib.Path(output_dir)
project_dir = output_path / "ultralytics"
cache_dir = output_path / "yolo-cache"
config_dir = output_path / "yolo-config"
matplotlib_dir = output_path / "matplotlib"
for directory in (project_dir, cache_dir, config_dir, matplotlib_dir):
directory.mkdir(parents=True, exist_ok=True)
os.environ.setdefault("YOLO_CONFIG_DIR", str(config_dir))
os.environ.setdefault("MPLCONFIGDIR", str(matplotlib_dir))
os.environ.setdefault("XDG_CONFIG_HOME", str(config_dir))
with contextlib.chdir(cache_dir):
result = YOLO(model_name).train(
data=str(data_yaml),
epochs=epochs,
imgsz=image_size,
batch=batch_size,
workers=workers,
project=str(project_dir),
name="train",
exist_ok=True,
)
weights_dir = project_dir / "train" / "weights"
best = weights_dir / "best.pt"
last = weights_dir / "last.pt"
checkpoint = best if best.exists() else last
if not checkpoint.exists():
raise RuntimeError("YOLO training did not produce a checkpoint")
copied_checkpoint = output_path / checkpoint.name
shutil.copy2(checkpoint, copied_checkpoint)
metrics = {
"operator": "YOLO 具身目标检测训练",
"modelFamily": "YOLO detection",
"dataYaml": str(data_yaml),
"model": model_name,
"epochs": epochs,
"imageSize": image_size,
"batchSize": batch_size,
"workers": workers,
"checkpoint": str(copied_checkpoint),
"results": str(getattr(result, "save_dir", project_dir / "train")),
}
return [
helpers["write_json_artifact"](run_id, node_id, output_path, "metrics.json", metrics),
helpers["build_artifact"](run_id, node_id, copied_checkpoint, "model"),
]
def _input_model_path(inputs: list[dict[str, Any]], output_dir: pathlib.Path) -> pathlib.Path | None:
for artifact in inputs:
artifact_type = str(artifact.get("type") or "").lower()
name = str(artifact.get("name") or artifact.get("filename") or "").lower()
raw_path = str(artifact.get("path") or artifact.get("uri") or "")
if not raw_path:
continue
candidate = pathlib.Path(raw_path)
if not candidate.exists():
continue
if artifact_type == "model" or name.endswith((".zip", ".pt", ".pth", ".safetensors", ".bin")):
if candidate.suffix.lower() == ".zip":
extract_dir = output_dir / "upstream-model"
if extract_dir.exists():
shutil.rmtree(extract_dir)
extract_dir.mkdir(parents=True, exist_ok=True)
with zipfile.ZipFile(candidate) as archive:
archive.extractall(extract_dir)
children = [item for item in extract_dir.iterdir() if item.is_dir()]
return children[0] if len(children) == 1 else extract_dir
return candidate
return None
def run_openvla_finetune(
run_id: str,
node_id: str,
output_dir: Any,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
helpers: HelperMap,
) -> list[dict[str, Any]]:
params = helpers["hyperparams"](node_data)
backend = _text(params, "TRAINING_BACKEND", "openvla_lora").lower()
if backend in {"transformers_text", "causal_lm"}:
return run_transformers_text_finetune(run_id, node_id, output_dir, node_data, inputs, helpers)
if backend != "openvla_lora":
raise RuntimeError("OpenVLA TRAINING_BACKEND must be openvla_lora or transformers_text")
openvla_root = pathlib.Path(_text(params, "OPENVLA_ROOT") or os.environ.get("OPENVLA_ROOT", "/opt/openvla"))
script = openvla_root / "vla-scripts" / "finetune.py"
if not script.exists():
raise RuntimeError(f"OpenVLA fine-tune script does not exist: {script}")
base_model = _text(params, "BASE_MODEL", "openvla/openvla-7b")
data_root = pathlib.Path(_text(params, "DATA_ROOT_DIR") or str(_dataset_path(params, inputs)))
if not data_root.exists():
raise RuntimeError(f"OpenVLA DATA_ROOT_DIR does not exist: {data_root}")
dataset_name = _text(params, "DATASET_NAME")
if not dataset_name:
raise RuntimeError("OpenVLA DATASET_NAME is required for RLDS fine-tune")
output_path = pathlib.Path(output_dir)
run_root = output_path / "openvla-runs"
adapter_dir = output_path / "openvla-adapter"
gpu_count = _int(params, "GPU_COUNT", 1)
command = [
"torchrun",
"--standalone",
"--nnodes",
"1",
"--nproc-per-node",
str(gpu_count),
str(script),
"--vla_path",
base_model,
"--data_root_dir",
str(data_root),
"--dataset_name",
dataset_name,
"--run_root_dir",
str(run_root),
"--adapter_tmp_dir",
str(adapter_dir),
"--batch_size",
str(_int(params, "BATCH_SIZE", 8)),
"--max_steps",
str(_int(params, "MAX_STEPS", 1000)),
"--save_steps",
str(_int(params, "SAVE_STEPS", 500)),
"--learning_rate",
str(_float(params, "LEARNING_RATE", 0.0005)),
"--lora_rank",
str(_int(params, "LORA_RANK", 32)),
"--lora_dropout",
str(_float(params, "LORA_DROPOUT", 0.0)),
"--use_lora",
"True",
"--use_quantization",
"True" if _bool(params, "USE_QUANTIZATION", False) else "False",
"--image_aug",
"True" if _bool(params, "IMAGE_AUG", True) else "False",
]
for override in _json_list(params, "OPENVLA_OVERRIDES_JSON"):
if not isinstance(override, str) or not override.strip() or override.strip().startswith("-"):
raise RuntimeError("OPENVLA_OVERRIDES_JSON entries must be non-empty draccus override strings")
command.append(override.strip())
_run_framework_command(
command,
cwd=openvla_root,
output_dir=output_path,
framework="OpenVLA LoRA fine-tune",
metadata={
"baseModel": base_model,
"dataRootDir": str(data_root),
"datasetName": dataset_name,
"trainingBackend": backend,
"gpuCount": gpu_count,
},
env={**os.environ, "TOKENIZERS_PARALLELISM": "false", "WANDB_MODE": os.environ.get("WANDB_MODE", "offline")},
)
if not adapter_dir.exists():
found_model_dir = next((path for path in run_root.rglob("*") if path.is_dir() and any(item.name == "adapter_config.json" for item in path.iterdir())), None)
if found_model_dir is not None:
adapter_dir = found_model_dir
archive_path = _zip_directory(adapter_dir, output_path / "openvla-adapter.zip")
metrics = {
"operator": "OpenVLA 微调训练",
"modelFamily": "OpenVLA LoRA",
"trainingBackend": backend,
"baseModel": base_model,
"dataRootDir": str(data_root),
"datasetName": dataset_name,
"runRootDir": str(run_root),
"adapterDir": str(adapter_dir),
"modelArchive": str(archive_path),
"gpuCount": gpu_count,
}
return _framework_artifacts(helpers, run_id, node_id, output_path, metrics, archive_path)
def run_openvla_inference(
run_id: str,
node_id: str,
output_dir: Any,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
helpers: HelperMap,
) -> list[dict[str, Any]]:
params = helpers["hyperparams"](node_data)
backend = _text(params, "INFERENCE_BACKEND", "openvla_transformers").lower()
output_path = pathlib.Path(output_dir)
upstream_model = _input_model_path(inputs, output_path)
if not _text(params, "MODEL_PATH") and upstream_model is not None:
node_data = dict(node_data)
hyperparams = [dict(item) for item in node_data.get("hyperparams", []) if isinstance(item, dict)]
hyperparams.append({"key": "MODEL_PATH", "value": str(upstream_model)})
node_data["hyperparams"] = hyperparams
if backend in {"openvla_transformers", "transformers_text", "causal_lm"}:
return run_transformers_text_inference(run_id, node_id, output_dir, node_data, inputs, helpers)
raise RuntimeError("OpenVLA INFERENCE_BACKEND must be openvla_transformers")
def run_transformers_text_finetune(
run_id: str,
node_id: str,
output_dir: Any,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
helpers: HelperMap,
) -> list[dict[str, Any]]:
try:
import torch
from torch.utils.data import DataLoader, Dataset
from transformers import AutoModelForCausalLM, AutoTokenizer
except Exception as exc: # pragma: no cover - exercised in container images
raise RuntimeError("Transformers and PyTorch are required in the OpenVLA fine-tune operator image") from exc
params = helpers["hyperparams"](node_data)
base_model = _text(params, "BASE_MODEL")
if not base_model:
raise RuntimeError("BASE_MODEL is required for OpenVLA fine-tune")
dataset_path = _dataset_path(params, inputs)
records = _load_json_records(dataset_path)
text_field = _text(params, "TEXT_FIELD", "instruction")
target_field = _text(params, "TARGET_FIELD", "action")
max_length = _int(params, "MAX_LENGTH", 256)
epochs = _int(params, "EPOCHS", 1)
learning_rate = _float(params, "LEARNING_RATE", 0.00005)
samples = [
f"{item.get(text_field, '')}\n{item.get(target_field, '')}".strip()
for item in records
if str(item.get(text_field, "")).strip() and str(item.get(target_field, "")).strip()
]
if not samples:
raise RuntimeError("OpenVLA fine-tune dataset must contain text/action samples")
output_path = pathlib.Path(output_dir)
cache_dir = _prepare_transformers_cache(params, output_path)
tokenizer = AutoTokenizer.from_pretrained(base_model, cache_dir=str(cache_dir))
if tokenizer.pad_token is None:
tokenizer.pad_token = tokenizer.eos_token
model = AutoModelForCausalLM.from_pretrained(base_model, cache_dir=str(cache_dir))
class TextDataset(Dataset):
def __len__(self) -> int:
return len(samples)
def __getitem__(self, index: int) -> dict[str, Any]:
encoded = tokenizer(samples[index], truncation=True, padding="max_length", max_length=max_length, return_tensors="pt")
item = {key: value.squeeze(0) for key, value in encoded.items()}
item["labels"] = item["input_ids"].clone()
return item
loader = DataLoader(TextDataset(), batch_size=min(_int(params, "BATCH_SIZE", 1), len(samples)), shuffle=True)
optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)
model.train()
losses: list[float] = []
for _ in range(epochs):
epoch_loss = 0.0
for batch in loader:
optimizer.zero_grad(set_to_none=True)
output = model(**batch)
output.loss.backward()
optimizer.step()
epoch_loss += float(output.loss.detach().cpu())
losses.append(epoch_loss / max(len(loader), 1))
model_dir = output_path / "openvla-adapter"
model.save_pretrained(model_dir)
tokenizer.save_pretrained(model_dir)
model_archive = output_path / "openvla-adapter.zip"
with zipfile.ZipFile(model_archive, "w", compression=zipfile.ZIP_DEFLATED) as archive:
for item in sorted(model_dir.rglob("*")):
if item.is_file():
archive.write(item, item.relative_to(output_path))
metrics = {
"operator": "OpenVLA 微调训练",
"modelFamily": "OpenVLA CausalLM fine-tune",
"baseModel": base_model,
"datasetPath": str(dataset_path),
"sampleCount": len(samples),
"epochs": epochs,
"finalLoss": losses[-1],
"losses": losses,
"modelDir": str(model_dir),
"modelArchive": str(model_archive),
}
return [
helpers["write_json_artifact"](run_id, node_id, output_path, "metrics.json", metrics),
helpers["build_artifact"](run_id, node_id, model_archive, "model"),
]
def run_transformers_text_inference(
run_id: str,
node_id: str,
output_dir: Any,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
helpers: HelperMap,
) -> list[dict[str, Any]]:
try:
from transformers import AutoModelForCausalLM, AutoTokenizer
except Exception as exc: # pragma: no cover - exercised in container images
raise RuntimeError("Transformers is required in the OpenVLA inference operator image") from exc
params = helpers["hyperparams"](node_data)
model_path = _text(params, "MODEL_PATH") or _text(params, "BASE_MODEL")
if not model_path:
raise RuntimeError("MODEL_PATH or BASE_MODEL is required for OpenVLA inference")
prompt = _text(params, "PROMPT")
if not prompt:
records = _load_json_records(_dataset_path(params, inputs))
prompt = str(records[0].get(_text(params, "TEXT_FIELD", "instruction"), "")).strip() if records else ""
if not prompt:
raise RuntimeError("OpenVLA inference requires PROMPT or an input sample instruction")
max_new_tokens = _int(params, "MAX_NEW_TOKENS", 64)
output_path = pathlib.Path(output_dir)
cache_dir = _prepare_transformers_cache(params, output_path)
tokenizer = AutoTokenizer.from_pretrained(model_path, cache_dir=str(cache_dir))
model = AutoModelForCausalLM.from_pretrained(model_path, cache_dir=str(cache_dir))
encoded = tokenizer(prompt, return_tensors="pt")
generated = model.generate(**encoded, max_new_tokens=max_new_tokens)
text = tokenizer.decode(generated[0], skip_special_tokens=True)
payload = {
"operator": "OpenVLA 推理",
"modelFamily": "OpenVLA CausalLM inference",
"modelPath": model_path,
"prompt": prompt,
"maxNewTokens": max_new_tokens,
"prediction": text,
}
return [helpers["write_json_artifact"](run_id, node_id, output_path, "inference-result.json", payload)]