Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions data-management/viewer/backend/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,20 @@ COPY pyproject.toml uv.lock ./
ENV UV_COMPILE_BYTECODE=1 \
UV_LINK_MODE=copy \
PATH=/app/.venv/bin:$PATH
RUN uv sync --frozen --no-dev --extra azure --extra analysis --extra export --extra auth --extra yolo
ARG AZURE_ONLY=false
RUN if [ "${AZURE_ONLY}" = "true" ]; then \
uv sync --frozen --no-dev --extra azure; \
else \
uv sync --frozen --no-dev --extra azure --extra analysis --extra export --extra auth --extra yolo; \
fi

# Copy application source
COPY src/ ./src/

# Run as an unprivileged user
RUN useradd --system --no-create-home appuser && chown -R appuser /app
RUN useradd --system --no-create-home appuser \
&& mkdir -p /data /models \
&& chown -R appuser /app /data /models
USER appuser

# Expose the API port (override with BACKEND_PORT env var at runtime)
Expand Down
6 changes: 2 additions & 4 deletions data-management/viewer/frontend/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,7 @@ WORKDIR /app

COPY package.json package-lock.json ./
COPY data-management/viewer/frontend/package.json data-management/viewer/frontend/package.json
RUN --mount=type=cache,target=/root/.npm \
--mount=type=secret,id=npmrc,target=/root/.npmrc \
npm ci --ignore-scripts --workspace robotic-training-data-tool --include-workspace-root=false
RUN npm ci --ignore-scripts --workspace robotic-training-data-tool --include-workspace-root=false

COPY data-management/viewer/frontend data-management/viewer/frontend
COPY data-management/viewer/specifications/language-tag-corpus.json data-management/viewer/specifications/language-tag-corpus.json
Expand Down Expand Up @@ -50,7 +48,7 @@ USER nginx

# localhost resolves to IPv6 [::1] first in this image, but nginx listens on
# IPv4 0.0.0.0:8080 only, so probe 127.0.0.1 explicitly.
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --start-interval=3s --retries=3 \
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
CMD wget -qO- http://127.0.0.1:8080/ >/dev/null 2>&1 || exit 1

CMD ["/bin/sh", "-c", "envsubst '${NGINX_BACKEND_HOST} ${NGINX_BACKEND_SCHEME} ${NGINX_RESOLVER}' < /etc/nginx/templates/default.conf.template > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;'"]
61 changes: 61 additions & 0 deletions training/vla/images/groot-n17/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
# syntax=docker/dockerfile:1
ARG BASE_IMAGE=pytorch/pytorch:2.6.0-cuda12.4-cudnn9-devel@sha256:0cf3402e946b7c384ba943ee05c90b4c5a4a05227923921f2b0918c011cfaf56
FROM ${BASE_IMAGE}

ARG ISAAC_GROOT_REF=23ace64f17aa5015259b8609d371eb61a357c776
ARG UV_VERSION=0.8.14

SHELL ["/bin/bash", "-Eeuo", "pipefail", "-c"]

ENV DEBIAN_FRONTEND=noninteractive \
GROOT_PREBUILT_ROOT=/opt/Isaac-GR00T \
UV_LINK_MODE=copy

RUN apt-get update -qq \
&& apt-get install -y --no-install-recommends \
ca-certificates \
build-essential \
cmake \
curl \
ffmpeg \
git \
git-lfs \
libgl1-mesa-glx \
libglib2.0-0 \
libsm6 \
libvulkan-dev \
libxext6 \
libxrender-dev \
python3.10-dev \
wget \
&& rm -rf /var/lib/apt/lists/*

RUN python3 -m pip install --no-cache-dir "uv==${UV_VERSION}"

RUN mkdir -p "${GROOT_PREBUILT_ROOT}" \
&& cd "${GROOT_PREBUILT_ROOT}" \
&& git init \
&& git remote add origin https://github.com/NVIDIA/Isaac-GR00T.git \
&& git fetch --depth 1 origin "${ISAAC_GROOT_REF}" \
&& git checkout --detach FETCH_HEAD \
&& test "$(git rev-parse HEAD)" = "${ISAAC_GROOT_REF}" \
&& git lfs install --system \
&& UV_PREVIEW=1 UV_HTTP_TIMEOUT=300 UV_CONCURRENT_DOWNLOADS=4 \
uv sync --frozen --no-install-project --no-dev --no-cache \
&& uv pip install --python "${GROOT_PREBUILT_ROOT}/.venv/bin/python" \
-e . --no-deps

COPY write-image-metadata.py /tmp/write-image-metadata.py
RUN "${GROOT_PREBUILT_ROOT}/.venv/bin/python" /tmp/write-image-metadata.py \
--isaac-groot-ref "${ISAAC_GROOT_REF}" \
--uv-version "${UV_VERSION}" \
--output /opt/groot-n17-image.json \
&& rm /tmp/write-image-metadata.py

ENV VIRTUAL_ENV=/opt/Isaac-GR00T/.venv
ENV PATH=/opt/Isaac-GR00T/.venv/bin:${PATH}

LABEL org.opencontainers.image.title="GR00T N1.7 Azure ML training runtime" \
org.opencontainers.image.description="Dependency-only runtime; model weights and credentials are mounted at job execution" \
org.opencontainers.image.source="https://github.com/NVIDIA/Isaac-GR00T" \
org.opencontainers.image.revision="${ISAAC_GROOT_REF}"
47 changes: 47 additions & 0 deletions training/vla/images/groot-n17/write-image-metadata.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
from __future__ import annotations

import argparse
import importlib.metadata
import json
from pathlib import Path

EXPECTED_RUNTIME = {
"accelerate": "1.13.0",
"flash-attn": "2.7.4.post1",
"numpy": "1.26.4",
"torch": "2.7.1+cu128",
"torchvision": "0.22.1+cu128",
}


def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--isaac-groot-ref", required=True)
parser.add_argument("--uv-version", required=True)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()

actual = {
name: importlib.metadata.version(name)
for name in EXPECTED_RUNTIME
}
if actual != EXPECTED_RUNTIME:
raise RuntimeError(f"locked runtime differs from Item 9: {actual}")

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Item 9 is task-relative and has no meaning to image builders or operators. Please state the runtime invariant directly and include the expected and actual version maps.

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 800a07d. The error now states the runtime invariant directly and includes both version maps: GR00T N1.7 runtime must match the pinned CUDA, PyTorch, and NumPy ABI; expected {...}, found {...}.


args.output.write_text(
json.dumps(
{
"isaac_groot_ref": args.isaac_groot_ref,
"runtime_versions": actual,
"uv_version": args.uv_version,
},
indent=2,
sort_keys=True,
)
+ "\n",
encoding="utf-8",
)


if __name__ == "__main__":
main()
141 changes: 141 additions & 0 deletions training/vla/scripts/groot/azureml-train-entry.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
#!/usr/bin/env bash
set -Eeuo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
TRAINING_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"

required=(
DATASET_PATH
OUTPUT_ROOT
AZURE_SUBSCRIPTION_ID
AZURE_RESOURCE_GROUP
AZUREML_WORKSPACE_NAME
)
for name in "${required[@]}"; do
if [ -z "${!name:-}" ]; then
echo "ERROR: required environment variable is empty: ${name}" >&2
exit 1
fi
done

if [ -z "${MODALITY_CONFIG_PATH:-}" ] && [ -z "${MODALITY_CONFIG_B64:-}" ]; then
echo "ERROR: MODALITY_CONFIG_PATH or MODALITY_CONFIG_B64 is required" >&2
exit 1
fi

if [ -n "${MODALITY_CONFIG_PATH:-}" ]; then
case "${MODALITY_CONFIG_PATH}" in
/*) ;;
*) MODALITY_CONFIG_PATH="${TRAINING_ROOT}/${MODALITY_CONFIG_PATH}" ;;
esac
export MODALITY_CONFIG_PATH
fi

DATASET_MOUNT="${DATASET_PATH}"
DATASET_OVERLAY="/tmp/azureml-dataset"
rm -rf "${DATASET_OVERLAY}"
mkdir -p "${DATASET_OVERLAY}"
for entry in "${DATASET_MOUNT}"/*; do
[ -e "${entry}" ] || continue
if [ "$(basename "${entry}")" = "meta" ]; then
cp -a "${entry}" "${DATASET_OVERLAY}/meta"
else
ln -s "${entry}" "${DATASET_OVERLAY}/$(basename "${entry}")"
fi
done
[ -d "${DATASET_OVERLAY}/meta" ] || {
echo "ERROR: mounted dataset has no meta directory" >&2
exit 1
}
export DATASET_PATH="${DATASET_OVERLAY}"
echo "[azureml] created writable metadata overlay for the read-only dataset mount"

if [ "${RESUME:-false}" = "true" ]; then
: "${RESUME_CHECKPOINT:?RESUME_CHECKPOINT is required when RESUME=true}"
[ -d "${RESUME_CHECKPOINT}" ] || {
echo "ERROR: resume checkpoint input is not a directory: ${RESUME_CHECKPOINT}" >&2
exit 1
}
latest_resume_checkpoint="$(
find "${RESUME_CHECKPOINT}" -type d -name 'checkpoint-*' |
sort -V |
tail -n 1
)"
[ -n "${latest_resume_checkpoint}" ] || {
echo "ERROR: resume checkpoint input contains no trainer_state.json" >&2
exit 1
}
[ -f "${latest_resume_checkpoint}/trainer_state.json" ] || {
echo "ERROR: latest resume checkpoint contains no trainer_state.json" >&2
exit 1
}
resume_output="${OUTPUT_ROOT}/${RUN_ID_OVERRIDE:-training}"
mkdir -p "${resume_output}"
ln -s "${latest_resume_checkpoint}" \
"${resume_output}/$(basename "${latest_resume_checkpoint}")"
echo "[azureml] linked resume checkpoint: ${latest_resume_checkpoint}"
fi

if [ -n "${HF_TOKEN_SECRET_URL:-}" ]; then
[ -z "${HF_TOKEN:-}" ] || {
echo "ERROR: HF_TOKEN and HF_TOKEN_SECRET_URL cannot both be set" >&2
exit 1
}
HF_TOKEN="$(python3 - <<'PY'
import json
import os
import urllib.parse
import urllib.request

identity_query = {
"api-version": "2018-02-01",
"resource": "https://vault.azure.net",
}
if os.environ.get("AZURE_CLIENT_ID"):
identity_query["client_id"] = os.environ["AZURE_CLIENT_ID"]
identity_url = (
"http://169.254.169.254/metadata/identity/oauth2/token?"

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This uses the VM IMDS endpoint, but Azure ML command jobs expose managed identity through the per-job MSI_ENDPOINT and MSI_SECRET. The recommended --hf-token-secret-url path will fail before training. Please obtain the Key Vault token through the Azure ML job MSI endpoint (or a credential implementation that uses it) and fail explicitly when those variables are unavailable.

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 800a07d. The token request now uses the Azure ML job endpoint, MSI_ENDPOINT with the secret: MSI_SECRET header and api-version=2017-09-01, matching azure-identity's AzureMLCredential. The clientid value comes from AZURE_CLIENT_ID or DEFAULT_IDENTITY_CLIENT_ID. The script fails with an explicit error when MSI_ENDPOINT or MSI_SECRET is unavailable.

+ urllib.parse.urlencode(identity_query)
)
identity_request = urllib.request.Request(identity_url, headers={"Metadata": "true"})
imds_opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
with imds_opener.open(identity_request, timeout=30) as response:
access_token = json.load(response)["access_token"]

secret_url = os.environ["HF_TOKEN_SECRET_URL"]
separator = "&" if "?" in secret_url else "?"
secret_request = urllib.request.Request(
secret_url + separator + "api-version=7.4",
headers={"Authorization": f"Bearer {access_token}"},
)
with urllib.request.urlopen(secret_request, timeout=30) as response:
secret_value = json.load(response)["value"]
if not secret_value:
raise RuntimeError("Hugging Face token secret is empty")
print(secret_value, end="")
PY
)"
export HF_TOKEN
echo "[azureml] loaded Hugging Face token from Key Vault"
fi

export DATA_CONFIG="${DATA_CONFIG:-custom_n17}"
export BASE_MODEL="${BASE_MODEL:-nvidia/GR00T-N1.7-3B}"
export BASE_MODEL_REVISION="${BASE_MODEL_REVISION:-2fc962b973bccdd5d8ce4f67cc63b264d6886495}"
export ISAAC_GROOT_REF="${ISAAC_GROOT_REF:-23ace64f17aa5015259b8609d371eb61a357c776}"
export EMBODIMENT_TAG="${EMBODIMENT_TAG:-NEW_EMBODIMENT}"
export BATCH_SIZE="${BATCH_SIZE:-4}"
export MAX_STEPS="${MAX_STEPS:-100}"
export SAVE_STEPS="${SAVE_STEPS:-50}"
export DATALOADER_WORKERS="${DATALOADER_WORKERS:-4}"
export NUM_GPUS="${NUM_GPUS:-2}"
export RUN_ID_OVERRIDE="${RUN_ID_OVERRIDE:-training}"
export AZURE_UPLOAD="${AZURE_UPLOAD:-false}"
export MODEL_CACHE_ROOT="${MODEL_CACHE_ROOT:-}"

if [ "${AZURE_UPLOAD}" = "true" ]; then
: "${AZUREML_MODEL_NAME:?AZUREML_MODEL_NAME is required when AZURE_UPLOAD=true}"
cp "${TRAINING_ROOT}/utils/aml_mirror.py" /tmp/aml_mirror.py
fi

exec "${SCRIPT_DIR}/osmo-train-entry.sh"
Loading
Loading