874 lines
35 KiB
Python
874 lines
35 KiB
Python
from __future__ import annotations
|
|
|
|
import csv
|
|
import contextlib
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
import pathlib
|
|
import shutil
|
|
import subprocess
|
|
import zipfile
|
|
from typing import Any, Callable
|
|
|
|
|
|
HelperMap = dict[str, Callable[..., Any]]
|
|
|
|
|
|
def _text(params: dict[str, str], key: str, default: str = "") -> str:
|
|
return str(params.get(key, default) or default).strip()
|
|
|
|
|
|
def _int(params: dict[str, str], key: str, default: int) -> int:
|
|
return _int_at_least(params, key, default, 1)
|
|
|
|
|
|
def _int_at_least(params: dict[str, str], key: str, default: int, minimum: int) -> int:
|
|
value = _text(params, key, str(default))
|
|
try:
|
|
parsed = int(value)
|
|
except ValueError as exc:
|
|
raise RuntimeError(f"{key} must be an integer") from exc
|
|
if parsed < minimum:
|
|
raise RuntimeError(f"{key} must be >= {minimum}")
|
|
return parsed
|
|
|
|
|
|
def _float(params: dict[str, str], key: str, default: float) -> float:
|
|
value = _text(params, key, str(default))
|
|
try:
|
|
parsed = float(value)
|
|
except ValueError as exc:
|
|
raise RuntimeError(f"{key} must be a number") from exc
|
|
if not math.isfinite(parsed):
|
|
raise RuntimeError(f"{key} must be finite")
|
|
return parsed
|
|
|
|
|
|
def _bool(params: dict[str, str], key: str, default: bool) -> bool:
|
|
value = _text(params, key, "true" if default else "false").lower()
|
|
if value in {"1", "true", "yes", "on"}:
|
|
return True
|
|
if value in {"0", "false", "no", "off"}:
|
|
return False
|
|
raise RuntimeError(f"{key} must be a boolean")
|
|
|
|
|
|
def _json_list(params: dict[str, str], key: str) -> list[Any]:
|
|
value = _text(params, key)
|
|
if not value:
|
|
return []
|
|
payload = json.loads(value)
|
|
if not isinstance(payload, list):
|
|
raise RuntimeError(f"{key} must be a JSON array")
|
|
return payload
|
|
|
|
|
|
def _dataset_path(params: dict[str, str], inputs: list[dict[str, Any]]) -> pathlib.Path:
|
|
explicit = _text(params, "DATASET_PATH") or _text(params, "TRAIN_DATA_PATH") or _text(params, "SOURCE_PATH")
|
|
if explicit:
|
|
path = pathlib.Path(explicit)
|
|
if not path.exists():
|
|
raise RuntimeError(f"dataset path does not exist: {path}")
|
|
return path
|
|
for artifact in inputs:
|
|
path = pathlib.Path(str(artifact.get("path") or artifact.get("uri") or ""))
|
|
if path.exists():
|
|
return path
|
|
raise RuntimeError("structured model operator requires DATASET_PATH/TRAIN_DATA_PATH/SOURCE_PATH or an input artifact path")
|
|
|
|
|
|
def _prepare_transformers_cache(params: dict[str, str], output_dir: pathlib.Path) -> pathlib.Path:
|
|
explicit_cache_dir = _text(params, "HF_CACHE_DIR") or _text(params, "TRANSFORMERS_CACHE_DIR")
|
|
if explicit_cache_dir:
|
|
cache_dir = pathlib.Path(explicit_cache_dir)
|
|
else:
|
|
cache_key = hashlib.sha256(str(output_dir).encode("utf-8")).hexdigest()[:16]
|
|
cache_dir = pathlib.Path("/tmp") / "droboticflow-hf-cache" / cache_key
|
|
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
if not os.access(cache_dir, os.W_OK):
|
|
raise RuntimeError(f"transformers cache directory is not writable: {cache_dir}")
|
|
os.environ["HF_HUB_DISABLE_SYMLINKS_WARNING"] = "1"
|
|
os.environ.setdefault("HF_HUB_DISABLE_XET", "1")
|
|
os.environ["HF_HOME"] = str(cache_dir)
|
|
os.environ["HUGGINGFACE_HUB_CACHE"] = str(cache_dir / "hub")
|
|
os.environ["TRANSFORMERS_CACHE"] = str(cache_dir / "transformers")
|
|
return cache_dir
|
|
|
|
|
|
def _find_first_file(root: pathlib.Path, suffixes: tuple[str, ...]) -> pathlib.Path | None:
|
|
if root.is_file() and root.suffix.lower() in suffixes:
|
|
return root
|
|
if not root.exists():
|
|
return None
|
|
candidates = [path for path in root.rglob("*") if path.is_file() and path.suffix.lower() in suffixes]
|
|
if not candidates:
|
|
return None
|
|
priority_names = ("best.pt", "latest.ckpt", "last.pt")
|
|
for name in priority_names:
|
|
for candidate in candidates:
|
|
if candidate.name == name:
|
|
return candidate
|
|
candidates.sort(key=lambda path: path.stat().st_mtime, reverse=True)
|
|
return candidates[0]
|
|
|
|
|
|
def _zip_directory(source_dir: pathlib.Path, archive_path: pathlib.Path) -> pathlib.Path:
|
|
if not source_dir.exists() or not source_dir.is_dir():
|
|
raise RuntimeError(f"model directory does not exist: {source_dir}")
|
|
with zipfile.ZipFile(archive_path, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
|
for item in sorted(source_dir.rglob("*")):
|
|
if item.is_file():
|
|
archive.write(item, item.relative_to(source_dir.parent))
|
|
return archive_path
|
|
|
|
|
|
def _run_framework_command(
|
|
command: list[str],
|
|
*,
|
|
cwd: pathlib.Path | None,
|
|
output_dir: pathlib.Path,
|
|
framework: str,
|
|
metadata: dict[str, Any],
|
|
env: dict[str, str] | None = None,
|
|
) -> dict[str, Any]:
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
started = dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z")
|
|
result = subprocess.run(
|
|
command,
|
|
cwd=str(cwd) if cwd else None,
|
|
env=env,
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
(output_dir / "framework-stdout.log").write_text(result.stdout or "", encoding="utf-8")
|
|
(output_dir / "framework-stderr.log").write_text(result.stderr or "", encoding="utf-8")
|
|
payload = {
|
|
"framework": framework,
|
|
"command": command,
|
|
"cwd": str(cwd) if cwd else "",
|
|
"returnCode": result.returncode,
|
|
"startedAt": started,
|
|
**metadata,
|
|
}
|
|
(output_dir / "framework-command.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
if result.returncode != 0:
|
|
raise RuntimeError(f"{framework} training command failed with exit code {result.returncode}; see framework-stderr.log")
|
|
return payload
|
|
|
|
|
|
def _framework_artifacts(
|
|
helpers: HelperMap,
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: pathlib.Path,
|
|
metrics: dict[str, Any],
|
|
model_path: pathlib.Path,
|
|
) -> list[dict[str, Any]]:
|
|
if not model_path.exists():
|
|
raise RuntimeError(f"expected framework model artifact was not produced: {model_path}")
|
|
return [
|
|
helpers["write_json_artifact"](run_id, node_id, output_dir, "metrics.json", metrics),
|
|
helpers["build_artifact"](run_id, node_id, output_dir / "framework-command.json", "json"),
|
|
helpers["build_artifact"](run_id, node_id, output_dir / "framework-stdout.log", "log"),
|
|
helpers["build_artifact"](run_id, node_id, output_dir / "framework-stderr.log", "log"),
|
|
helpers["build_artifact"](run_id, node_id, model_path, "model"),
|
|
]
|
|
|
|
|
|
def _load_json_records(path: pathlib.Path) -> list[dict[str, Any]]:
|
|
if path.is_dir():
|
|
candidates = [
|
|
path / "samples.jsonl",
|
|
path / "samples.json",
|
|
path / "dataset.jsonl",
|
|
path / "dataset.json",
|
|
path / "manifest.jsonl",
|
|
path / "manifest.json",
|
|
]
|
|
for candidate in candidates:
|
|
if candidate.exists():
|
|
return _load_json_records(candidate)
|
|
raise RuntimeError(f"dataset directory does not contain a supported JSON/JSONL manifest: {path}")
|
|
if path.suffix.lower() == ".jsonl":
|
|
records: list[dict[str, Any]] = []
|
|
for line in path.read_text(encoding="utf-8").splitlines():
|
|
if line.strip():
|
|
payload = json.loads(line)
|
|
if isinstance(payload, dict):
|
|
records.append(payload)
|
|
return records
|
|
if path.suffix.lower() == ".csv":
|
|
with path.open(newline="", encoding="utf-8") as handle:
|
|
return [dict(row) for row in csv.DictReader(handle)]
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
if isinstance(payload, dict):
|
|
for key in ("samples", "items", "records", "data"):
|
|
value = payload.get(key)
|
|
if isinstance(value, list):
|
|
return [item for item in value if isinstance(item, dict)]
|
|
return [payload]
|
|
if isinstance(payload, list):
|
|
return [item for item in payload if isinstance(item, dict)]
|
|
raise RuntimeError(f"dataset must be a JSON object/list, JSONL, or CSV: {path}")
|
|
|
|
|
|
def _vector(value: Any) -> list[float]:
|
|
if isinstance(value, str):
|
|
try:
|
|
value = json.loads(value)
|
|
except json.JSONDecodeError:
|
|
value = [float(part) for part in value.replace(";", ",").split(",") if part.strip()]
|
|
if isinstance(value, (int, float)):
|
|
value = [value]
|
|
if not isinstance(value, list) or not value:
|
|
raise RuntimeError("state/action fields must be numeric arrays")
|
|
vector = [float(item) for item in value]
|
|
if not all(math.isfinite(item) for item in vector):
|
|
raise RuntimeError("state/action vectors must contain finite values")
|
|
return vector
|
|
|
|
|
|
def _field_value(item: dict[str, Any], field: str) -> Any:
|
|
value: Any = item
|
|
for part in field.split("."):
|
|
if isinstance(value, dict):
|
|
value = value.get(part)
|
|
else:
|
|
return None
|
|
return value
|
|
|
|
|
|
def run_policy_training(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: Any,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
helpers: HelperMap,
|
|
*,
|
|
operator_name: str,
|
|
model_family: str,
|
|
) -> list[dict[str, Any]]:
|
|
try:
|
|
import torch
|
|
from torch import nn
|
|
from torch.utils.data import DataLoader, TensorDataset
|
|
except Exception as exc: # pragma: no cover - exercised in container images
|
|
raise RuntimeError("PyTorch is required in the operator image for structured policy training") from exc
|
|
|
|
params = helpers["hyperparams"](node_data)
|
|
dataset_path = _dataset_path(params, inputs)
|
|
records = _load_json_records(dataset_path)
|
|
if len(records) < 2:
|
|
raise RuntimeError("policy training requires at least 2 real samples")
|
|
state_field = _text(params, "STATE_FIELD", "state")
|
|
action_field = _text(params, "ACTION_FIELD", "action")
|
|
states = [_vector(_field_value(item, state_field)) for item in records]
|
|
actions = [_vector(_field_value(item, action_field)) for item in records]
|
|
state_dim = len(states[0])
|
|
action_dim = len(actions[0])
|
|
if any(len(item) != state_dim for item in states) or any(len(item) != action_dim for item in actions):
|
|
raise RuntimeError("all state/action vectors must have consistent dimensions")
|
|
|
|
epochs = _int(params, "EPOCHS", 5)
|
|
batch_size = _int(params, "BATCH_SIZE", 8)
|
|
learning_rate = _float(params, "LEARNING_RATE", 0.001)
|
|
hidden_dim = _int(params, "HIDDEN_DIM", 128)
|
|
chunk_size = _int(params, "CHUNK_SIZE", 1)
|
|
|
|
x = torch.tensor(states, dtype=torch.float32)
|
|
y = torch.tensor(actions, dtype=torch.float32)
|
|
loader = DataLoader(TensorDataset(x, y), batch_size=min(batch_size, len(records)), shuffle=True)
|
|
model = nn.Sequential(nn.Linear(state_dim, hidden_dim), nn.ReLU(), nn.Linear(hidden_dim, action_dim))
|
|
optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)
|
|
loss_fn = nn.MSELoss()
|
|
losses: list[float] = []
|
|
for _ in range(epochs):
|
|
epoch_loss = 0.0
|
|
for batch_x, batch_y in loader:
|
|
optimizer.zero_grad(set_to_none=True)
|
|
loss = loss_fn(model(batch_x), batch_y)
|
|
loss.backward()
|
|
optimizer.step()
|
|
epoch_loss += float(loss.detach().cpu())
|
|
losses.append(epoch_loss / max(len(loader), 1))
|
|
|
|
output_path = pathlib.Path(output_dir)
|
|
checkpoint_path = output_path / "checkpoint.pt"
|
|
torch.save(
|
|
{
|
|
"operator": operator_name,
|
|
"modelFamily": model_family,
|
|
"stateField": state_field,
|
|
"actionField": action_field,
|
|
"stateDim": state_dim,
|
|
"actionDim": action_dim,
|
|
"chunkSize": chunk_size,
|
|
"modelStateDict": model.state_dict(),
|
|
},
|
|
checkpoint_path,
|
|
)
|
|
metrics = {
|
|
"operator": operator_name,
|
|
"modelFamily": model_family,
|
|
"trainingBackend": "torch_mlp",
|
|
"datasetPath": str(dataset_path),
|
|
"sampleCount": len(records),
|
|
"stateDim": state_dim,
|
|
"actionDim": action_dim,
|
|
"epochs": epochs,
|
|
"batchSize": batch_size,
|
|
"learningRate": learning_rate,
|
|
"chunkSize": chunk_size,
|
|
"finalLoss": losses[-1],
|
|
"losses": losses,
|
|
}
|
|
return [
|
|
helpers["write_json_artifact"](run_id, node_id, output_path, "metrics.json", metrics),
|
|
helpers["build_artifact"](run_id, node_id, checkpoint_path, "model"),
|
|
]
|
|
|
|
|
|
def run_lerobot_act_training(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: Any,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
helpers: HelperMap,
|
|
) -> list[dict[str, Any]]:
|
|
params = helpers["hyperparams"](node_data)
|
|
backend = _text(params, "TRAINING_BACKEND", "lerobot").lower()
|
|
if backend == "torch_mlp":
|
|
return run_policy_training(
|
|
run_id,
|
|
node_id,
|
|
output_dir,
|
|
node_data,
|
|
inputs,
|
|
helpers,
|
|
operator_name="ACT 策略训练",
|
|
model_family="ACT",
|
|
)
|
|
if backend != "lerobot":
|
|
raise RuntimeError("ACT TRAINING_BACKEND must be lerobot or torch_mlp")
|
|
|
|
dataset_repo_id = _text(params, "DATASET_REPO_ID")
|
|
dataset_root = _text(params, "DATASET_PATH") or _text(params, "TRAIN_DATA_PATH")
|
|
if not dataset_repo_id and not dataset_root:
|
|
raise RuntimeError("ACT LeRobot training requires DATASET_REPO_ID or DATASET_PATH")
|
|
if dataset_root and not pathlib.Path(dataset_root).exists():
|
|
raise RuntimeError(f"ACT DATASET_PATH does not exist: {dataset_root}")
|
|
|
|
output_path = pathlib.Path(output_dir)
|
|
train_dir = output_path / "lerobot-act"
|
|
command = [
|
|
_text(params, "LEROBOT_TRAIN_BIN", "lerobot-train"),
|
|
f"--policy.type={_text(params, 'POLICY_TYPE', 'act')}",
|
|
f"--output_dir={train_dir}",
|
|
f"--job_name={_text(params, 'JOB_NAME', f'droboticflow-act-{run_id[:8]}')}",
|
|
f"--policy.device={_text(params, 'DEVICE', 'cuda')}",
|
|
f"--wandb.enable={str(_bool(params, 'WANDB_ENABLE', False)).lower()}",
|
|
]
|
|
if dataset_repo_id:
|
|
command.append(f"--dataset.repo_id={dataset_repo_id}")
|
|
if dataset_root:
|
|
command.append(f"--dataset.root={dataset_root}")
|
|
policy_repo_id = _text(params, "POLICY_REPO_ID")
|
|
if policy_repo_id:
|
|
command.append(f"--policy.repo_id={policy_repo_id}")
|
|
if _text(params, "MAX_STEPS"):
|
|
command.append(f"--steps={_int(params, 'MAX_STEPS', 1000)}")
|
|
if _text(params, "BATCH_SIZE"):
|
|
command.append(f"--batch_size={_int(params, 'BATCH_SIZE', 8)}")
|
|
if _text(params, "LEARNING_RATE"):
|
|
command.append(f"--optimizer.lr={_float(params, 'LEARNING_RATE', 0.0001)}")
|
|
if _text(params, "CHUNK_SIZE"):
|
|
command.append(f"--policy.chunk_size={_int(params, 'CHUNK_SIZE', 100)}")
|
|
for override in _json_list(params, "LEROBOT_OVERRIDES_JSON"):
|
|
if not isinstance(override, str) or not override.strip() or override.strip().startswith("-"):
|
|
raise RuntimeError("LEROBOT_OVERRIDES_JSON entries must be non-empty Hydra override strings")
|
|
command.append(override.strip())
|
|
|
|
_run_framework_command(
|
|
command,
|
|
cwd=None,
|
|
output_dir=output_path,
|
|
framework="LeRobot ACT",
|
|
metadata={
|
|
"datasetRepoId": dataset_repo_id,
|
|
"datasetRoot": dataset_root,
|
|
"trainingBackend": backend,
|
|
},
|
|
env={**os.environ, "HF_HUB_DISABLE_XET": os.environ.get("HF_HUB_DISABLE_XET", "1")},
|
|
)
|
|
checkpoint = _find_first_file(train_dir, (".safetensors", ".pt", ".pth", ".bin"))
|
|
if checkpoint is None:
|
|
raise RuntimeError("LeRobot ACT training completed but no checkpoint file was found")
|
|
copied_checkpoint = output_path / checkpoint.name
|
|
if checkpoint != copied_checkpoint:
|
|
shutil.copy2(checkpoint, copied_checkpoint)
|
|
metrics = {
|
|
"operator": "ACT 策略训练",
|
|
"modelFamily": "LeRobot ACT",
|
|
"trainingBackend": backend,
|
|
"datasetRepoId": dataset_repo_id,
|
|
"datasetRoot": dataset_root,
|
|
"outputDir": str(train_dir),
|
|
"checkpoint": str(copied_checkpoint),
|
|
"device": _text(params, "DEVICE", "cuda"),
|
|
}
|
|
return _framework_artifacts(helpers, run_id, node_id, output_path, metrics, copied_checkpoint)
|
|
|
|
|
|
def run_diffusion_policy_training(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: Any,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
helpers: HelperMap,
|
|
) -> list[dict[str, Any]]:
|
|
params = helpers["hyperparams"](node_data)
|
|
backend = _text(params, "TRAINING_BACKEND", "diffusion_policy").lower()
|
|
if backend == "torch_mlp":
|
|
return run_policy_training(
|
|
run_id,
|
|
node_id,
|
|
output_dir,
|
|
node_data,
|
|
inputs,
|
|
helpers,
|
|
operator_name="Diffusion Policy 训练",
|
|
model_family="Diffusion Policy",
|
|
)
|
|
if backend != "diffusion_policy":
|
|
raise RuntimeError("Diffusion Policy TRAINING_BACKEND must be diffusion_policy or torch_mlp")
|
|
|
|
repo_root = pathlib.Path(_text(params, "DIFFUSION_POLICY_ROOT") or os.environ.get("DIFFUSION_POLICY_ROOT", "/opt/diffusion_policy"))
|
|
train_py = repo_root / "train.py"
|
|
if not train_py.exists():
|
|
raise RuntimeError(f"Diffusion Policy train.py does not exist: {train_py}")
|
|
dataset_path = _dataset_path(params, inputs)
|
|
output_path = pathlib.Path(output_dir)
|
|
train_dir = output_path / "diffusion-policy"
|
|
config_name = _text(params, "CONFIG_NAME", "image_pusht_diffusion_policy_cnn")
|
|
command = [
|
|
"python",
|
|
str(train_py),
|
|
"--config-dir",
|
|
_text(params, "CONFIG_DIR", "."),
|
|
f"--config-name={config_name}",
|
|
f"hydra.run.dir={train_dir}",
|
|
f"training.device={_text(params, 'DEVICE', 'cuda:0')}",
|
|
]
|
|
task_name = _text(params, "TASK_NAME")
|
|
if task_name:
|
|
command.append(f"task={task_name}")
|
|
command.extend(
|
|
[
|
|
f"task.dataset.zarr_path={dataset_path}",
|
|
f"training.num_epochs={_int(params, 'EPOCHS', 1)}",
|
|
f"dataloader.batch_size={_int(params, 'BATCH_SIZE', 8)}",
|
|
f"optimizer.lr={_float(params, 'LEARNING_RATE', 0.0001)}",
|
|
]
|
|
)
|
|
if _text(params, "HORIZON"):
|
|
command.append(f"horizon={_int(params, 'HORIZON', 16)}")
|
|
if _text(params, "DIFFUSION_STEPS"):
|
|
command.append(f"policy.num_inference_steps={_int(params, 'DIFFUSION_STEPS', 32)}")
|
|
for override in _json_list(params, "HYDRA_OVERRIDES_JSON"):
|
|
if not isinstance(override, str) or not override.strip() or override.strip().startswith("-"):
|
|
raise RuntimeError("HYDRA_OVERRIDES_JSON entries must be non-empty Hydra override strings")
|
|
command.append(override.strip())
|
|
|
|
_run_framework_command(
|
|
command,
|
|
cwd=repo_root,
|
|
output_dir=output_path,
|
|
framework="real-stanford Diffusion Policy",
|
|
metadata={
|
|
"datasetPath": str(dataset_path),
|
|
"trainingBackend": backend,
|
|
"configName": config_name,
|
|
"taskName": task_name,
|
|
},
|
|
env={**os.environ, "WANDB_MODE": os.environ.get("WANDB_MODE", "offline")},
|
|
)
|
|
checkpoint = _find_first_file(train_dir, (".ckpt", ".pt", ".pth"))
|
|
if checkpoint is None:
|
|
raise RuntimeError("Diffusion Policy training completed but no checkpoint file was found")
|
|
copied_checkpoint = output_path / checkpoint.name
|
|
if checkpoint != copied_checkpoint:
|
|
shutil.copy2(checkpoint, copied_checkpoint)
|
|
metrics = {
|
|
"operator": "Diffusion Policy 训练",
|
|
"modelFamily": "real-stanford Diffusion Policy",
|
|
"trainingBackend": backend,
|
|
"datasetPath": str(dataset_path),
|
|
"configName": config_name,
|
|
"taskName": task_name,
|
|
"outputDir": str(train_dir),
|
|
"checkpoint": str(copied_checkpoint),
|
|
"device": _text(params, "DEVICE", "cuda:0"),
|
|
}
|
|
return _framework_artifacts(helpers, run_id, node_id, output_path, metrics, copied_checkpoint)
|
|
|
|
|
|
def run_yolo_training(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: Any,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
helpers: HelperMap,
|
|
) -> list[dict[str, Any]]:
|
|
try:
|
|
from ultralytics import YOLO
|
|
except Exception as exc: # pragma: no cover - exercised in container images
|
|
raise RuntimeError("Ultralytics is required in the YOLO operator image") from exc
|
|
|
|
params = helpers["hyperparams"](node_data)
|
|
data_yaml = pathlib.Path(_text(params, "DATA_YAML") or _text(params, "DATASET_YAML") or str(_dataset_path(params, inputs)))
|
|
if not data_yaml.exists():
|
|
raise RuntimeError(f"YOLO data.yaml does not exist: {data_yaml}")
|
|
model_name = _text(params, "MODEL_SIZE", "yolo11n.pt")
|
|
epochs = _int(params, "EPOCHS", 10)
|
|
image_size = _int(params, "IMAGE_SIZE", 640)
|
|
batch_size = _int(params, "BATCH_SIZE", 8)
|
|
workers = _int_at_least(params, "WORKERS", 0, 0)
|
|
output_path = pathlib.Path(output_dir)
|
|
project_dir = output_path / "ultralytics"
|
|
cache_dir = output_path / "yolo-cache"
|
|
config_dir = output_path / "yolo-config"
|
|
matplotlib_dir = output_path / "matplotlib"
|
|
for directory in (project_dir, cache_dir, config_dir, matplotlib_dir):
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
os.environ.setdefault("YOLO_CONFIG_DIR", str(config_dir))
|
|
os.environ.setdefault("MPLCONFIGDIR", str(matplotlib_dir))
|
|
os.environ.setdefault("XDG_CONFIG_HOME", str(config_dir))
|
|
with contextlib.chdir(cache_dir):
|
|
result = YOLO(model_name).train(
|
|
data=str(data_yaml),
|
|
epochs=epochs,
|
|
imgsz=image_size,
|
|
batch=batch_size,
|
|
workers=workers,
|
|
project=str(project_dir),
|
|
name="train",
|
|
exist_ok=True,
|
|
)
|
|
weights_dir = project_dir / "train" / "weights"
|
|
best = weights_dir / "best.pt"
|
|
last = weights_dir / "last.pt"
|
|
checkpoint = best if best.exists() else last
|
|
if not checkpoint.exists():
|
|
raise RuntimeError("YOLO training did not produce a checkpoint")
|
|
copied_checkpoint = output_path / checkpoint.name
|
|
shutil.copy2(checkpoint, copied_checkpoint)
|
|
metrics = {
|
|
"operator": "YOLO 具身目标检测训练",
|
|
"modelFamily": "YOLO detection",
|
|
"dataYaml": str(data_yaml),
|
|
"model": model_name,
|
|
"epochs": epochs,
|
|
"imageSize": image_size,
|
|
"batchSize": batch_size,
|
|
"workers": workers,
|
|
"checkpoint": str(copied_checkpoint),
|
|
"results": str(getattr(result, "save_dir", project_dir / "train")),
|
|
}
|
|
return [
|
|
helpers["write_json_artifact"](run_id, node_id, output_path, "metrics.json", metrics),
|
|
helpers["build_artifact"](run_id, node_id, copied_checkpoint, "model"),
|
|
]
|
|
|
|
|
|
def _input_model_path(inputs: list[dict[str, Any]], output_dir: pathlib.Path) -> pathlib.Path | None:
|
|
for artifact in inputs:
|
|
artifact_type = str(artifact.get("type") or "").lower()
|
|
name = str(artifact.get("name") or artifact.get("filename") or "").lower()
|
|
raw_path = str(artifact.get("path") or artifact.get("uri") or "")
|
|
if not raw_path:
|
|
continue
|
|
candidate = pathlib.Path(raw_path)
|
|
if not candidate.exists():
|
|
continue
|
|
if artifact_type == "model" or name.endswith((".zip", ".pt", ".pth", ".safetensors", ".bin")):
|
|
if candidate.suffix.lower() == ".zip":
|
|
extract_dir = output_dir / "upstream-model"
|
|
if extract_dir.exists():
|
|
shutil.rmtree(extract_dir)
|
|
extract_dir.mkdir(parents=True, exist_ok=True)
|
|
with zipfile.ZipFile(candidate) as archive:
|
|
archive.extractall(extract_dir)
|
|
children = [item for item in extract_dir.iterdir() if item.is_dir()]
|
|
return children[0] if len(children) == 1 else extract_dir
|
|
return candidate
|
|
return None
|
|
|
|
|
|
def run_openvla_finetune(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: Any,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
helpers: HelperMap,
|
|
) -> list[dict[str, Any]]:
|
|
params = helpers["hyperparams"](node_data)
|
|
backend = _text(params, "TRAINING_BACKEND", "openvla_lora").lower()
|
|
if backend in {"transformers_text", "causal_lm"}:
|
|
return run_transformers_text_finetune(run_id, node_id, output_dir, node_data, inputs, helpers)
|
|
if backend != "openvla_lora":
|
|
raise RuntimeError("OpenVLA TRAINING_BACKEND must be openvla_lora or transformers_text")
|
|
|
|
openvla_root = pathlib.Path(_text(params, "OPENVLA_ROOT") or os.environ.get("OPENVLA_ROOT", "/opt/openvla"))
|
|
script = openvla_root / "vla-scripts" / "finetune.py"
|
|
if not script.exists():
|
|
raise RuntimeError(f"OpenVLA fine-tune script does not exist: {script}")
|
|
base_model = _text(params, "BASE_MODEL", "openvla/openvla-7b")
|
|
data_root = pathlib.Path(_text(params, "DATA_ROOT_DIR") or str(_dataset_path(params, inputs)))
|
|
if not data_root.exists():
|
|
raise RuntimeError(f"OpenVLA DATA_ROOT_DIR does not exist: {data_root}")
|
|
dataset_name = _text(params, "DATASET_NAME")
|
|
if not dataset_name:
|
|
raise RuntimeError("OpenVLA DATASET_NAME is required for RLDS fine-tune")
|
|
output_path = pathlib.Path(output_dir)
|
|
run_root = output_path / "openvla-runs"
|
|
adapter_dir = output_path / "openvla-adapter"
|
|
gpu_count = _int(params, "GPU_COUNT", 1)
|
|
command = [
|
|
"torchrun",
|
|
"--standalone",
|
|
"--nnodes",
|
|
"1",
|
|
"--nproc-per-node",
|
|
str(gpu_count),
|
|
str(script),
|
|
"--vla_path",
|
|
base_model,
|
|
"--data_root_dir",
|
|
str(data_root),
|
|
"--dataset_name",
|
|
dataset_name,
|
|
"--run_root_dir",
|
|
str(run_root),
|
|
"--adapter_tmp_dir",
|
|
str(adapter_dir),
|
|
"--batch_size",
|
|
str(_int(params, "BATCH_SIZE", 8)),
|
|
"--max_steps",
|
|
str(_int(params, "MAX_STEPS", 1000)),
|
|
"--save_steps",
|
|
str(_int(params, "SAVE_STEPS", 500)),
|
|
"--learning_rate",
|
|
str(_float(params, "LEARNING_RATE", 0.0005)),
|
|
"--lora_rank",
|
|
str(_int(params, "LORA_RANK", 32)),
|
|
"--lora_dropout",
|
|
str(_float(params, "LORA_DROPOUT", 0.0)),
|
|
"--use_lora",
|
|
"True",
|
|
"--use_quantization",
|
|
"True" if _bool(params, "USE_QUANTIZATION", False) else "False",
|
|
"--image_aug",
|
|
"True" if _bool(params, "IMAGE_AUG", True) else "False",
|
|
]
|
|
for override in _json_list(params, "OPENVLA_OVERRIDES_JSON"):
|
|
if not isinstance(override, str) or not override.strip() or override.strip().startswith("-"):
|
|
raise RuntimeError("OPENVLA_OVERRIDES_JSON entries must be non-empty draccus override strings")
|
|
command.append(override.strip())
|
|
|
|
_run_framework_command(
|
|
command,
|
|
cwd=openvla_root,
|
|
output_dir=output_path,
|
|
framework="OpenVLA LoRA fine-tune",
|
|
metadata={
|
|
"baseModel": base_model,
|
|
"dataRootDir": str(data_root),
|
|
"datasetName": dataset_name,
|
|
"trainingBackend": backend,
|
|
"gpuCount": gpu_count,
|
|
},
|
|
env={**os.environ, "TOKENIZERS_PARALLELISM": "false", "WANDB_MODE": os.environ.get("WANDB_MODE", "offline")},
|
|
)
|
|
if not adapter_dir.exists():
|
|
found_model_dir = next((path for path in run_root.rglob("*") if path.is_dir() and any(item.name == "adapter_config.json" for item in path.iterdir())), None)
|
|
if found_model_dir is not None:
|
|
adapter_dir = found_model_dir
|
|
archive_path = _zip_directory(adapter_dir, output_path / "openvla-adapter.zip")
|
|
metrics = {
|
|
"operator": "OpenVLA 微调训练",
|
|
"modelFamily": "OpenVLA LoRA",
|
|
"trainingBackend": backend,
|
|
"baseModel": base_model,
|
|
"dataRootDir": str(data_root),
|
|
"datasetName": dataset_name,
|
|
"runRootDir": str(run_root),
|
|
"adapterDir": str(adapter_dir),
|
|
"modelArchive": str(archive_path),
|
|
"gpuCount": gpu_count,
|
|
}
|
|
return _framework_artifacts(helpers, run_id, node_id, output_path, metrics, archive_path)
|
|
|
|
|
|
def run_openvla_inference(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: Any,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
helpers: HelperMap,
|
|
) -> list[dict[str, Any]]:
|
|
params = helpers["hyperparams"](node_data)
|
|
backend = _text(params, "INFERENCE_BACKEND", "openvla_transformers").lower()
|
|
output_path = pathlib.Path(output_dir)
|
|
upstream_model = _input_model_path(inputs, output_path)
|
|
if not _text(params, "MODEL_PATH") and upstream_model is not None:
|
|
node_data = dict(node_data)
|
|
hyperparams = [dict(item) for item in node_data.get("hyperparams", []) if isinstance(item, dict)]
|
|
hyperparams.append({"key": "MODEL_PATH", "value": str(upstream_model)})
|
|
node_data["hyperparams"] = hyperparams
|
|
if backend in {"openvla_transformers", "transformers_text", "causal_lm"}:
|
|
return run_transformers_text_inference(run_id, node_id, output_dir, node_data, inputs, helpers)
|
|
raise RuntimeError("OpenVLA INFERENCE_BACKEND must be openvla_transformers")
|
|
|
|
|
|
def run_transformers_text_finetune(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: Any,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
helpers: HelperMap,
|
|
) -> list[dict[str, Any]]:
|
|
try:
|
|
import torch
|
|
from torch.utils.data import DataLoader, Dataset
|
|
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
except Exception as exc: # pragma: no cover - exercised in container images
|
|
raise RuntimeError("Transformers and PyTorch are required in the OpenVLA fine-tune operator image") from exc
|
|
|
|
params = helpers["hyperparams"](node_data)
|
|
base_model = _text(params, "BASE_MODEL")
|
|
if not base_model:
|
|
raise RuntimeError("BASE_MODEL is required for OpenVLA fine-tune")
|
|
dataset_path = _dataset_path(params, inputs)
|
|
records = _load_json_records(dataset_path)
|
|
text_field = _text(params, "TEXT_FIELD", "instruction")
|
|
target_field = _text(params, "TARGET_FIELD", "action")
|
|
max_length = _int(params, "MAX_LENGTH", 256)
|
|
epochs = _int(params, "EPOCHS", 1)
|
|
learning_rate = _float(params, "LEARNING_RATE", 0.00005)
|
|
samples = [
|
|
f"{item.get(text_field, '')}\n{item.get(target_field, '')}".strip()
|
|
for item in records
|
|
if str(item.get(text_field, "")).strip() and str(item.get(target_field, "")).strip()
|
|
]
|
|
if not samples:
|
|
raise RuntimeError("OpenVLA fine-tune dataset must contain text/action samples")
|
|
|
|
output_path = pathlib.Path(output_dir)
|
|
cache_dir = _prepare_transformers_cache(params, output_path)
|
|
tokenizer = AutoTokenizer.from_pretrained(base_model, cache_dir=str(cache_dir))
|
|
if tokenizer.pad_token is None:
|
|
tokenizer.pad_token = tokenizer.eos_token
|
|
model = AutoModelForCausalLM.from_pretrained(base_model, cache_dir=str(cache_dir))
|
|
|
|
class TextDataset(Dataset):
|
|
def __len__(self) -> int:
|
|
return len(samples)
|
|
|
|
def __getitem__(self, index: int) -> dict[str, Any]:
|
|
encoded = tokenizer(samples[index], truncation=True, padding="max_length", max_length=max_length, return_tensors="pt")
|
|
item = {key: value.squeeze(0) for key, value in encoded.items()}
|
|
item["labels"] = item["input_ids"].clone()
|
|
return item
|
|
|
|
loader = DataLoader(TextDataset(), batch_size=min(_int(params, "BATCH_SIZE", 1), len(samples)), shuffle=True)
|
|
optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)
|
|
model.train()
|
|
losses: list[float] = []
|
|
for _ in range(epochs):
|
|
epoch_loss = 0.0
|
|
for batch in loader:
|
|
optimizer.zero_grad(set_to_none=True)
|
|
output = model(**batch)
|
|
output.loss.backward()
|
|
optimizer.step()
|
|
epoch_loss += float(output.loss.detach().cpu())
|
|
losses.append(epoch_loss / max(len(loader), 1))
|
|
|
|
model_dir = output_path / "openvla-adapter"
|
|
model.save_pretrained(model_dir)
|
|
tokenizer.save_pretrained(model_dir)
|
|
model_archive = output_path / "openvla-adapter.zip"
|
|
with zipfile.ZipFile(model_archive, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
|
for item in sorted(model_dir.rglob("*")):
|
|
if item.is_file():
|
|
archive.write(item, item.relative_to(output_path))
|
|
metrics = {
|
|
"operator": "OpenVLA 微调训练",
|
|
"modelFamily": "OpenVLA CausalLM fine-tune",
|
|
"baseModel": base_model,
|
|
"datasetPath": str(dataset_path),
|
|
"sampleCount": len(samples),
|
|
"epochs": epochs,
|
|
"finalLoss": losses[-1],
|
|
"losses": losses,
|
|
"modelDir": str(model_dir),
|
|
"modelArchive": str(model_archive),
|
|
}
|
|
return [
|
|
helpers["write_json_artifact"](run_id, node_id, output_path, "metrics.json", metrics),
|
|
helpers["build_artifact"](run_id, node_id, model_archive, "model"),
|
|
]
|
|
|
|
|
|
def run_transformers_text_inference(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: Any,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
helpers: HelperMap,
|
|
) -> list[dict[str, Any]]:
|
|
try:
|
|
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
except Exception as exc: # pragma: no cover - exercised in container images
|
|
raise RuntimeError("Transformers is required in the OpenVLA inference operator image") from exc
|
|
|
|
params = helpers["hyperparams"](node_data)
|
|
model_path = _text(params, "MODEL_PATH") or _text(params, "BASE_MODEL")
|
|
if not model_path:
|
|
raise RuntimeError("MODEL_PATH or BASE_MODEL is required for OpenVLA inference")
|
|
prompt = _text(params, "PROMPT")
|
|
if not prompt:
|
|
records = _load_json_records(_dataset_path(params, inputs))
|
|
prompt = str(records[0].get(_text(params, "TEXT_FIELD", "instruction"), "")).strip() if records else ""
|
|
if not prompt:
|
|
raise RuntimeError("OpenVLA inference requires PROMPT or an input sample instruction")
|
|
max_new_tokens = _int(params, "MAX_NEW_TOKENS", 64)
|
|
output_path = pathlib.Path(output_dir)
|
|
cache_dir = _prepare_transformers_cache(params, output_path)
|
|
tokenizer = AutoTokenizer.from_pretrained(model_path, cache_dir=str(cache_dir))
|
|
model = AutoModelForCausalLM.from_pretrained(model_path, cache_dir=str(cache_dir))
|
|
encoded = tokenizer(prompt, return_tensors="pt")
|
|
generated = model.generate(**encoded, max_new_tokens=max_new_tokens)
|
|
text = tokenizer.decode(generated[0], skip_special_tokens=True)
|
|
payload = {
|
|
"operator": "OpenVLA 推理",
|
|
"modelFamily": "OpenVLA CausalLM inference",
|
|
"modelPath": model_path,
|
|
"prompt": prompt,
|
|
"maxNewTokens": max_new_tokens,
|
|
"prediction": text,
|
|
}
|
|
return [helpers["write_json_artifact"](run_id, node_id, output_path, "inference-result.json", payload)]
|