build: pin DISCOVERSE archive supply chain
This commit is contained in:
parent
30a3a0ebfb
commit
b120ecbaac
@ -1,10 +1,10 @@
|
||||
FROM pytorch/pytorch:2.2.0-cuda12.1-cudnn8-runtime
|
||||
|
||||
ARG BUILD_ID=dev
|
||||
ARG SOURCE_COMMIT=unknown
|
||||
ARG ANYSPLAT_COMMIT=5f5e208a7dd57d52e43ea0d553a95eab526e8775
|
||||
ARG DISCOVERSE_COMMIT=d67f47c084aba0e0cf422a8725235f8b9238655a
|
||||
ARG ACT_COMMIT=f8ab39c8d829d85571ab6d0250412ba4d0698fa0
|
||||
ARG DISCOVERSE_ARCHIVE_SHA256=c8fbd6ca571ce3d44a61c6d3903de956075ee28d29afdfbcfe2b3fcc4cdab350
|
||||
ARG ACT_ARCHIVE_SHA256=35c5dfc416ba0a5d7f10ad61252fa9f685ac4cf2b123cd6f1b3db1619a2b1b37
|
||||
ARG MOTRIXSIM_DOCS_COMMIT=2beecea67c81d25e29cefa30a80c0ddad7b44dde
|
||||
ARG VGGT_MODEL_REVISION=860abec7937da0a4c03c41d3c269c366e82abdf9
|
||||
|
||||
@ -27,14 +27,13 @@ RUN sed -i 's|http://archive.ubuntu.com/ubuntu|http://mirrors.aliyun.com/ubuntu|
|
||||
|
||||
WORKDIR /app
|
||||
COPY pyproject.toml README.md ./
|
||||
COPY src ./src
|
||||
COPY scripts ./scripts
|
||||
COPY vendor/gs_playground /opt/gs_playground
|
||||
COPY docker/anysplat-runtime-requirements.txt /tmp/anysplat-runtime-requirements.txt
|
||||
COPY docker/worker-constraints.txt /tmp/worker-constraints.txt
|
||||
|
||||
RUN python -m pip install --no-cache-dir --upgrade pip setuptools wheel && \
|
||||
python -m pip install --no-cache-dir -c /tmp/worker-constraints.txt . && \
|
||||
python -m pip install --no-cache-dir tomli==2.2.1 && \
|
||||
python -c 'import tomli; config=tomli.load(open("pyproject.toml", "rb")); open("/tmp/platform-requirements.txt", "w").write("\n".join(config["project"]["dependencies"]) + "\n")' && \
|
||||
python -m pip install --no-cache-dir -c /tmp/worker-constraints.txt -r /tmp/platform-requirements.txt && \
|
||||
python -m pip install --no-cache-dir torchvision==0.17.0 torchaudio==2.2.0 --index-url https://download.pytorch.org/whl/cu121 && \
|
||||
python -m pip install --no-cache-dir -c /tmp/worker-constraints.txt -r /tmp/anysplat-runtime-requirements.txt && \
|
||||
python -m pip install --no-cache-dir xformers==0.0.24 --no-deps && \
|
||||
@ -42,16 +41,31 @@ RUN python -m pip install --no-cache-dir --upgrade pip setuptools wheel && \
|
||||
python -m pip install --no-cache-dir 'https://github.com/nerfstudio-project/gsplat/releases/download/v1.4.0/gsplat-1.4.0%2Bpt22cu121-cp310-cp310-linux_x86_64.whl' && \
|
||||
python -m pip install --no-cache-dir -c /tmp/worker-constraints.txt motrixsim-core==0.8.0
|
||||
|
||||
COPY src ./src
|
||||
COPY scripts ./scripts
|
||||
COPY vendor/gs_playground /opt/gs_playground
|
||||
RUN python -m pip install --no-cache-dir --no-deps .
|
||||
|
||||
RUN git clone --filter=blob:none --no-checkout https://github.com/InternRobotics/AnySplat.git /opt/real2sim_paper_runtime/AnySplat && \
|
||||
git -C /opt/real2sim_paper_runtime/AnySplat checkout "${ANYSPLAT_COMMIT}" && \
|
||||
test "$(git -C /opt/real2sim_paper_runtime/AnySplat rev-parse HEAD)" = "${ANYSPLAT_COMMIT}" && \
|
||||
sed -i 's/VGGT.from_pretrained("facebook\/VGGT-1B")/VGGT.from_pretrained("facebook\/VGGT-1B", revision=os.environ.get("REAL2SIM_VGGT_MODEL_REVISION"))/g' /opt/real2sim_paper_runtime/AnySplat/src/model/encoder/anysplat.py
|
||||
|
||||
RUN git clone --filter=blob:none --no-checkout https://github.com/eust-w/DISCOVERSE.git /opt/DISCOVERSE && \
|
||||
git -C /opt/DISCOVERSE checkout "${DISCOVERSE_COMMIT}" && \
|
||||
git -C /opt/DISCOVERSE submodule update --init policies/act && \
|
||||
test "$(git -C /opt/DISCOVERSE rev-parse HEAD)" = "${DISCOVERSE_COMMIT}" && \
|
||||
test "$(git -C /opt/DISCOVERSE/policies/act rev-parse HEAD)" = "${ACT_COMMIT}" && \
|
||||
RUN mkdir -p /opt/DISCOVERSE /opt/DISCOVERSE/policies/act && \
|
||||
curl -fL --retry 5 --retry-all-errors --connect-timeout 20 --max-time 900 \
|
||||
-o /tmp/discoverse.tar.gz "https://codeload.github.com/eust-w/DISCOVERSE/tar.gz/${DISCOVERSE_COMMIT}" && \
|
||||
echo "${DISCOVERSE_ARCHIVE_SHA256} /tmp/discoverse.tar.gz" | sha256sum -c - && \
|
||||
tar -xzf /tmp/discoverse.tar.gz --strip-components=1 -C /opt/DISCOVERSE && \
|
||||
rm -rf /opt/DISCOVERSE/policies/act && mkdir -p /opt/DISCOVERSE/policies/act && \
|
||||
curl -fL --retry 5 --retry-all-errors --connect-timeout 20 --max-time 300 \
|
||||
-o /tmp/act.tar.gz "https://codeload.github.com/OpenGHz/Imitate-All/tar.gz/${ACT_COMMIT}" && \
|
||||
echo "${ACT_ARCHIVE_SHA256} /tmp/act.tar.gz" | sha256sum -c - && \
|
||||
tar -xzf /tmp/act.tar.gz --strip-components=1 -C /opt/DISCOVERSE/policies/act && \
|
||||
printf '{"schema":"real2sim.source.v1","source":"https://codeload.github.com/eust-w/DISCOVERSE/tar.gz/%s","commit":"%s","archive_sha256":"%s"}\n' \
|
||||
"${DISCOVERSE_COMMIT}" "${DISCOVERSE_COMMIT}" "${DISCOVERSE_ARCHIVE_SHA256}" > /opt/DISCOVERSE/.real2sim-source.json && \
|
||||
printf '{"schema":"real2sim.source.v1","source":"https://codeload.github.com/OpenGHz/Imitate-All/tar.gz/%s","commit":"%s","archive_sha256":"%s"}\n' \
|
||||
"${ACT_COMMIT}" "${ACT_COMMIT}" "${ACT_ARCHIVE_SHA256}" > /opt/DISCOVERSE/policies/act/.real2sim-source.json && \
|
||||
rm -f /tmp/discoverse.tar.gz /tmp/act.tar.gz && \
|
||||
grep -Ev '^[[:space:]]*(torchvision|torch|opencv-python)([[:space:]]|$)' /opt/DISCOVERSE/policies/act/requirements/train_eval.txt > /tmp/discoverse-act-train-requirements.txt && \
|
||||
python -m pip install --no-cache-dir -c /tmp/worker-constraints.txt -r /tmp/discoverse-act-train-requirements.txt && \
|
||||
python /app/scripts/prepare_discoverse_act_runtime.py --act-root /opt/DISCOVERSE/policies/act --expected-commit "${ACT_COMMIT}" && \
|
||||
@ -73,6 +87,8 @@ RUN python -c 'import mujoco, motrixsim, torch; from huggingface_hub import snap
|
||||
PYTHONPATH=/opt/real2sim_paper_runtime/AnySplat python -c 'from src.model.model.anysplat import AnySplat; assert AnySplat is not None' && \
|
||||
test -f /opt/gs_playground/scripts/real2sim_video_motrix_scene.py
|
||||
|
||||
ARG BUILD_ID=dev
|
||||
ARG SOURCE_COMMIT=unknown
|
||||
ENV REAL2SIM_PLATFORM_BUILD_ID=${BUILD_ID} \
|
||||
REAL2SIM_SOURCE_COMMIT=${SOURCE_COMMIT} \
|
||||
REAL2SIM_MOTRIXSIM_DOCS_COMMIT=${MOTRIXSIM_DOCS_COMMIT} \
|
||||
|
||||
@ -3,9 +3,10 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from real2sim_platform.source_revision import read_source_revision
|
||||
|
||||
|
||||
OPTIONAL_IMPORT = "from airbot_data_collection.tools.av_coder import AvCoder"
|
||||
OPTIONAL_REPLACEMENT = '''try:
|
||||
@ -28,7 +29,7 @@ def main() -> int:
|
||||
parser.add_argument("--expected-commit", required=True)
|
||||
args = parser.parse_args()
|
||||
root = args.act_root.resolve()
|
||||
commit = subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"], text=True).strip()
|
||||
commit = read_source_revision(root)
|
||||
if commit != args.expected_commit:
|
||||
raise SystemExit(f"ACT revision mismatch: expected {args.expected_commit}, got {commit}")
|
||||
target = root / "utils" / "utils.py"
|
||||
|
||||
@ -17,6 +17,7 @@ from urllib.parse import quote, unquote, urlparse
|
||||
from urllib.request import urlopen
|
||||
|
||||
from .config import PlatformConfig
|
||||
from .source_revision import read_source_revision
|
||||
from .storage import ALLOWED_AREAS, BosStorage, normalize_relative_path
|
||||
from .store import PlatformStore
|
||||
|
||||
@ -2016,10 +2017,7 @@ def _file_sha256(path: Path) -> str:
|
||||
|
||||
|
||||
def _git_head(root: Path) -> str:
|
||||
try:
|
||||
return subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL, timeout=5).strip()
|
||||
except Exception:
|
||||
return ""
|
||||
return read_source_revision(root)
|
||||
|
||||
|
||||
def _inspect_act_dataset(dataset_path: Path) -> dict[str, Any]:
|
||||
|
||||
@ -27,6 +27,7 @@ from . import __version__
|
||||
from .adapters import SPARK_VENDOR_PATH, collect_task_reports, enrich_scene_manifest_assets, run_doctor, scan_dataset
|
||||
from .auth import AuthManager, AuthenticationError, SESSION_COOKIE, validate_password, validate_role, validate_username
|
||||
from .config import PlatformConfig, ensure_workspace, is_path_allowed, load_config, write_default_config
|
||||
from .source_revision import read_source_revision
|
||||
from .storage import BosStorage, StorageBoundaryError, StorageUnavailable, safe_object_name, sha256_file
|
||||
from .store import PlatformStore
|
||||
from .worker import TaskWorker
|
||||
@ -1409,12 +1410,7 @@ def _capabilities(cfg: PlatformConfig, storage: BosStorage, store: PlatformStore
|
||||
|
||||
|
||||
def _git_commit_full(root: Path) -> str:
|
||||
if not root.is_dir():
|
||||
return ""
|
||||
try:
|
||||
return subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL, timeout=3).strip()
|
||||
except Exception:
|
||||
return ""
|
||||
return read_source_revision(root)
|
||||
|
||||
|
||||
def _module_available(name: str) -> bool:
|
||||
|
||||
49
src/real2sim_platform/source_revision.py
Normal file
49
src/real2sim_platform/source_revision.py
Normal file
@ -0,0 +1,49 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
SOURCE_MARKER = ".real2sim-source.json"
|
||||
_COMMIT_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||
_SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
|
||||
|
||||
|
||||
def read_source_revision(root: Path) -> str:
|
||||
"""Return the pinned revision from Git metadata or a verified archive marker."""
|
||||
|
||||
root = root.resolve()
|
||||
if not root.is_dir():
|
||||
return ""
|
||||
try:
|
||||
commit = subprocess.check_output(
|
||||
["git", "-C", str(root), "rev-parse", "HEAD"],
|
||||
text=True,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=5,
|
||||
).strip().lower()
|
||||
if _COMMIT_RE.fullmatch(commit):
|
||||
return commit
|
||||
except Exception:
|
||||
pass
|
||||
marker = read_source_marker(root)
|
||||
return str(marker.get("commit", ""))
|
||||
|
||||
|
||||
def read_source_marker(root: Path) -> dict[str, Any]:
|
||||
marker_path = root.resolve() / SOURCE_MARKER
|
||||
try:
|
||||
marker = json.loads(marker_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError, TypeError):
|
||||
return {}
|
||||
if marker.get("schema") != "real2sim.source.v1":
|
||||
return {}
|
||||
commit = str(marker.get("commit", "")).lower()
|
||||
archive_sha256 = str(marker.get("archive_sha256", "")).lower()
|
||||
source = str(marker.get("source", ""))
|
||||
if not _COMMIT_RE.fullmatch(commit) or not _SHA256_RE.fullmatch(archive_sha256) or not source.startswith("https://"):
|
||||
return {}
|
||||
return {"schema": marker["schema"], "source": source, "commit": commit, "archive_sha256": archive_sha256}
|
||||
@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import concurrent.futures
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
@ -15,6 +16,7 @@ from real2sim_platform.adapters import _act_task_config_source, index_artifacts_
|
||||
from real2sim_platform.rm65_controller import _write_preview_video, collect
|
||||
from real2sim_platform.runtime_gate import _motrixsim_assets_probe, runtime_key
|
||||
from real2sim_platform.server import create_app
|
||||
from real2sim_platform.source_revision import read_source_revision
|
||||
from real2sim_platform.storage import BosStorage, StorageBoundaryError, normalize_relative_path
|
||||
from real2sim_platform.store import PlatformStore
|
||||
|
||||
@ -350,6 +352,41 @@ def test_runtime_key_changes_with_deployed_build(tmp_path: Path, monkeypatch: py
|
||||
assert runtime_key(config) != first
|
||||
|
||||
|
||||
def test_source_revision_accepts_audited_archive_marker(tmp_path: Path) -> None:
|
||||
root = tmp_path / "archive-source"
|
||||
root.mkdir()
|
||||
commit = "d67f47c084aba0e0cf422a8725235f8b9238655a"
|
||||
(root / ".real2sim-source.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema": "real2sim.source.v1",
|
||||
"source": f"https://codeload.github.com/eust-w/DISCOVERSE/tar.gz/{commit}",
|
||||
"commit": commit,
|
||||
"archive_sha256": "c8fbd6ca571ce3d44a61c6d3903de956075ee28d29afdfbcfe2b3fcc4cdab350",
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
assert read_source_revision(root) == commit
|
||||
|
||||
|
||||
def test_source_revision_rejects_unverified_archive_marker(tmp_path: Path) -> None:
|
||||
root = tmp_path / "archive-source"
|
||||
root.mkdir()
|
||||
(root / ".real2sim-source.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema": "real2sim.source.v1",
|
||||
"source": "http://untrusted.invalid/archive.tar.gz",
|
||||
"commit": "not-a-commit",
|
||||
"archive_sha256": "not-a-checksum",
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
assert read_source_revision(root) == ""
|
||||
|
||||
|
||||
def test_motrixsim_runtime_scene_uses_current_python_when_module_is_installed(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
script_path = ROOT / "scripts/generate_motrixsim_runtime_scene.py"
|
||||
spec = importlib.util.spec_from_file_location("real2sim_generate_motrixsim_runtime_scene", script_path)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user