Repository navigation
feat(training): add GR00T N1.7 Azure ML smoke workflow #1712
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from 1 commit
34d80f2
1c2f65d
c9c3eab
800a07d
cfb9108
b35e577
6688943
a9368ef
e5ef1f1
56d53ba
ccc01fa
5d85e50
0f2222c
a8cd6eb
b4ea729
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| # syntax=docker/dockerfile:1 | ||
| ARG BASE_IMAGE=pytorch/pytorch:2.6.0-cuda12.4-cudnn9-devel@sha256:0cf3402e946b7c384ba943ee05c90b4c5a4a05227923921f2b0918c011cfaf56 | ||
| FROM ${BASE_IMAGE} | ||
|
|
||
| ARG ISAAC_GROOT_REF=23ace64f17aa5015259b8609d371eb61a357c776 | ||
| ARG UV_VERSION=0.8.14 | ||
|
|
||
| SHELL ["/bin/bash", "-Eeuo", "pipefail", "-c"] | ||
|
|
||
| ENV DEBIAN_FRONTEND=noninteractive \ | ||
| GROOT_PREBUILT_ROOT=/opt/Isaac-GR00T \ | ||
| UV_LINK_MODE=copy | ||
|
|
||
| RUN apt-get update -qq \ | ||
| && apt-get install -y --no-install-recommends \ | ||
| ca-certificates \ | ||
| build-essential \ | ||
| cmake \ | ||
| curl \ | ||
| ffmpeg \ | ||
| git \ | ||
| git-lfs \ | ||
| libgl1-mesa-glx \ | ||
| libglib2.0-0 \ | ||
| libsm6 \ | ||
| libvulkan-dev \ | ||
| libxext6 \ | ||
| libxrender-dev \ | ||
| python3.10-dev \ | ||
| wget \ | ||
| && rm -rf /var/lib/apt/lists/* | ||
|
|
||
| RUN python3 -m pip install --no-cache-dir "uv==${UV_VERSION}" | ||
|
|
||
| RUN mkdir -p "${GROOT_PREBUILT_ROOT}" \ | ||
| && cd "${GROOT_PREBUILT_ROOT}" \ | ||
| && git init \ | ||
| && git remote add origin https://github.com/NVIDIA/Isaac-GR00T.git \ | ||
| && git fetch --depth 1 origin "${ISAAC_GROOT_REF}" \ | ||
| && git checkout --detach FETCH_HEAD \ | ||
| && test "$(git rev-parse HEAD)" = "${ISAAC_GROOT_REF}" \ | ||
| && git lfs install --system \ | ||
| && UV_PREVIEW=1 UV_HTTP_TIMEOUT=300 UV_CONCURRENT_DOWNLOADS=4 \ | ||
| uv sync --frozen --no-install-project --no-dev --no-cache \ | ||
| && uv pip install --python "${GROOT_PREBUILT_ROOT}/.venv/bin/python" \ | ||
| -e . --no-deps | ||
|
|
||
| COPY write-image-metadata.py /tmp/write-image-metadata.py | ||
| RUN "${GROOT_PREBUILT_ROOT}/.venv/bin/python" /tmp/write-image-metadata.py \ | ||
| --isaac-groot-ref "${ISAAC_GROOT_REF}" \ | ||
| --uv-version "${UV_VERSION}" \ | ||
| --output /opt/groot-n17-image.json \ | ||
| && rm /tmp/write-image-metadata.py | ||
|
|
||
| ENV VIRTUAL_ENV=/opt/Isaac-GR00T/.venv | ||
| ENV PATH=/opt/Isaac-GR00T/.venv/bin:${PATH} | ||
|
|
||
| LABEL org.opencontainers.image.title="GR00T N1.7 Azure ML training runtime" \ | ||
| org.opencontainers.image.description="Dependency-only runtime; model weights and credentials are mounted at job execution" \ | ||
| org.opencontainers.image.source="https://github.com/NVIDIA/Isaac-GR00T" \ | ||
| org.opencontainers.image.revision="${ISAAC_GROOT_REF}" |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| from __future__ import annotations | ||
|
|
||
| import argparse | ||
| import importlib.metadata | ||
| import json | ||
| from pathlib import Path | ||
|
|
||
| EXPECTED_RUNTIME = { | ||
| "accelerate": "1.13.0", | ||
| "flash-attn": "2.7.4.post1", | ||
| "numpy": "1.26.4", | ||
| "torch": "2.7.1+cu128", | ||
| "torchvision": "0.22.1+cu128", | ||
| } | ||
|
|
||
|
|
||
| def main() -> None: | ||
| parser = argparse.ArgumentParser() | ||
| parser.add_argument("--isaac-groot-ref", required=True) | ||
| parser.add_argument("--uv-version", required=True) | ||
| parser.add_argument("--output", type=Path, required=True) | ||
| args = parser.parse_args() | ||
|
|
||
| actual = { | ||
| name: importlib.metadata.version(name) | ||
| for name in EXPECTED_RUNTIME | ||
| } | ||
| if actual != EXPECTED_RUNTIME: | ||
| raise RuntimeError(f"locked runtime differs from Item 9: {actual}") | ||
|
|
||
| args.output.write_text( | ||
| json.dumps( | ||
| { | ||
| "isaac_groot_ref": args.isaac_groot_ref, | ||
| "runtime_versions": actual, | ||
| "uv_version": args.uv_version, | ||
| }, | ||
| indent=2, | ||
| sort_keys=True, | ||
| ) | ||
| + "\n", | ||
| encoding="utf-8", | ||
| ) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| main() | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,141 @@ | ||
| #!/usr/bin/env bash | ||
| set -Eeuo pipefail | ||
|
|
||
| SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" | ||
| TRAINING_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" | ||
|
|
||
| required=( | ||
| DATASET_PATH | ||
| OUTPUT_ROOT | ||
| AZURE_SUBSCRIPTION_ID | ||
| AZURE_RESOURCE_GROUP | ||
| AZUREML_WORKSPACE_NAME | ||
| ) | ||
| for name in "${required[@]}"; do | ||
| if [ -z "${!name:-}" ]; then | ||
| echo "ERROR: required environment variable is empty: ${name}" >&2 | ||
| exit 1 | ||
| fi | ||
| done | ||
|
|
||
| if [ -z "${MODALITY_CONFIG_PATH:-}" ] && [ -z "${MODALITY_CONFIG_B64:-}" ]; then | ||
| echo "ERROR: MODALITY_CONFIG_PATH or MODALITY_CONFIG_B64 is required" >&2 | ||
| exit 1 | ||
| fi | ||
|
|
||
| if [ -n "${MODALITY_CONFIG_PATH:-}" ]; then | ||
| case "${MODALITY_CONFIG_PATH}" in | ||
| /*) ;; | ||
| *) MODALITY_CONFIG_PATH="${TRAINING_ROOT}/${MODALITY_CONFIG_PATH}" ;; | ||
| esac | ||
| export MODALITY_CONFIG_PATH | ||
| fi | ||
|
|
||
| DATASET_MOUNT="${DATASET_PATH}" | ||
| DATASET_OVERLAY="/tmp/azureml-dataset" | ||
| rm -rf "${DATASET_OVERLAY}" | ||
| mkdir -p "${DATASET_OVERLAY}" | ||
| for entry in "${DATASET_MOUNT}"/*; do | ||
| [ -e "${entry}" ] || continue | ||
| if [ "$(basename "${entry}")" = "meta" ]; then | ||
| cp -a "${entry}" "${DATASET_OVERLAY}/meta" | ||
| else | ||
| ln -s "${entry}" "${DATASET_OVERLAY}/$(basename "${entry}")" | ||
| fi | ||
| done | ||
| [ -d "${DATASET_OVERLAY}/meta" ] || { | ||
| echo "ERROR: mounted dataset has no meta directory" >&2 | ||
| exit 1 | ||
| } | ||
| export DATASET_PATH="${DATASET_OVERLAY}" | ||
| echo "[azureml] created writable metadata overlay for the read-only dataset mount" | ||
|
|
||
| if [ "${RESUME:-false}" = "true" ]; then | ||
| : "${RESUME_CHECKPOINT:?RESUME_CHECKPOINT is required when RESUME=true}" | ||
| [ -d "${RESUME_CHECKPOINT}" ] || { | ||
| echo "ERROR: resume checkpoint input is not a directory: ${RESUME_CHECKPOINT}" >&2 | ||
| exit 1 | ||
| } | ||
| latest_resume_checkpoint="$( | ||
| find "${RESUME_CHECKPOINT}" -type d -name 'checkpoint-*' | | ||
| sort -V | | ||
| tail -n 1 | ||
| )" | ||
| [ -n "${latest_resume_checkpoint}" ] || { | ||
| echo "ERROR: resume checkpoint input contains no trainer_state.json" >&2 | ||
| exit 1 | ||
| } | ||
| [ -f "${latest_resume_checkpoint}/trainer_state.json" ] || { | ||
| echo "ERROR: latest resume checkpoint contains no trainer_state.json" >&2 | ||
| exit 1 | ||
| } | ||
| resume_output="${OUTPUT_ROOT}/${RUN_ID_OVERRIDE:-training}" | ||
| mkdir -p "${resume_output}" | ||
| ln -s "${latest_resume_checkpoint}" \ | ||
| "${resume_output}/$(basename "${latest_resume_checkpoint}")" | ||
| echo "[azureml] linked resume checkpoint: ${latest_resume_checkpoint}" | ||
| fi | ||
|
|
||
| if [ -n "${HF_TOKEN_SECRET_URL:-}" ]; then | ||
| [ -z "${HF_TOKEN:-}" ] || { | ||
| echo "ERROR: HF_TOKEN and HF_TOKEN_SECRET_URL cannot both be set" >&2 | ||
| exit 1 | ||
| } | ||
| HF_TOKEN="$(python3 - <<'PY' | ||
| import json | ||
| import os | ||
| import urllib.parse | ||
| import urllib.request | ||
|
|
||
| identity_query = { | ||
| "api-version": "2018-02-01", | ||
| "resource": "https://vault.azure.net", | ||
| } | ||
| if os.environ.get("AZURE_CLIENT_ID"): | ||
| identity_query["client_id"] = os.environ["AZURE_CLIENT_ID"] | ||
| identity_url = ( | ||
| "http://169.254.169.254/metadata/identity/oauth2/token?" | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This uses the VM IMDS endpoint, but Azure ML command jobs expose managed identity through the per-job
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fixed in |
||
| + urllib.parse.urlencode(identity_query) | ||
| ) | ||
| identity_request = urllib.request.Request(identity_url, headers={"Metadata": "true"}) | ||
| imds_opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) | ||
| with imds_opener.open(identity_request, timeout=30) as response: | ||
| access_token = json.load(response)["access_token"] | ||
|
|
||
| secret_url = os.environ["HF_TOKEN_SECRET_URL"] | ||
| separator = "&" if "?" in secret_url else "?" | ||
| secret_request = urllib.request.Request( | ||
| secret_url + separator + "api-version=7.4", | ||
| headers={"Authorization": f"Bearer {access_token}"}, | ||
| ) | ||
| with urllib.request.urlopen(secret_request, timeout=30) as response: | ||
| secret_value = json.load(response)["value"] | ||
| if not secret_value: | ||
| raise RuntimeError("Hugging Face token secret is empty") | ||
| print(secret_value, end="") | ||
| PY | ||
| )" | ||
| export HF_TOKEN | ||
| echo "[azureml] loaded Hugging Face token from Key Vault" | ||
| fi | ||
|
|
||
| export DATA_CONFIG="${DATA_CONFIG:-custom_n17}" | ||
| export BASE_MODEL="${BASE_MODEL:-nvidia/GR00T-N1.7-3B}" | ||
| export BASE_MODEL_REVISION="${BASE_MODEL_REVISION:-2fc962b973bccdd5d8ce4f67cc63b264d6886495}" | ||
| export ISAAC_GROOT_REF="${ISAAC_GROOT_REF:-23ace64f17aa5015259b8609d371eb61a357c776}" | ||
| export EMBODIMENT_TAG="${EMBODIMENT_TAG:-NEW_EMBODIMENT}" | ||
| export BATCH_SIZE="${BATCH_SIZE:-4}" | ||
| export MAX_STEPS="${MAX_STEPS:-100}" | ||
| export SAVE_STEPS="${SAVE_STEPS:-50}" | ||
| export DATALOADER_WORKERS="${DATALOADER_WORKERS:-4}" | ||
| export NUM_GPUS="${NUM_GPUS:-2}" | ||
| export RUN_ID_OVERRIDE="${RUN_ID_OVERRIDE:-training}" | ||
| export AZURE_UPLOAD="${AZURE_UPLOAD:-false}" | ||
| export MODEL_CACHE_ROOT="${MODEL_CACHE_ROOT:-}" | ||
|
|
||
| if [ "${AZURE_UPLOAD}" = "true" ]; then | ||
| : "${AZUREML_MODEL_NAME:?AZUREML_MODEL_NAME is required when AZURE_UPLOAD=true}" | ||
| cp "${TRAINING_ROOT}/utils/aml_mirror.py" /tmp/aml_mirror.py | ||
| fi | ||
|
|
||
| exec "${SCRIPT_DIR}/osmo-train-entry.sh" | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Item 9is task-relative and has no meaning to image builders or operators. Please state the runtime invariant directly and include the expected and actual version maps.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Fixed in
800a07d. The error now states the runtime invariant directly and includes both version maps:GR00T N1.7 runtime must match the pinned CUDA, PyTorch, and NumPy ABI; expected {...}, found {...}.