Skip to content
Open
Show file tree
Hide file tree
Changes from 11 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions scripts/update-image-digests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ azureml_pin_specs=(
'DEFAULT_LEROBOT_EVAL_IMAGE|lerobot-inference-env|evaluation/sil/workflows/azureml/lerobot-eval.yaml'
'DEFAULT_LEROBOT_EVAL_IMAGE|lerobot-inference-env|evaluation/sil/workflows/azureml/vla-pi0-eval.yaml'
'DEFAULT_LEROBOT_TRAIN_IMAGE|vla-pi0-training-env|training/vla/workflows/azureml/vla-pi0-train.yaml'
'DEFAULT_GROOT_IMAGE|groot-n17-training-env|training/vla/workflows/azureml/groot-n17-train.yaml'
'DEFAULT_AZUREML_SMOKE_IMAGE|gpu-smoke-env|training/smoke/workflows/azureml/gpu-smoke.yaml'
)
azureml_environment_line_re="^[[:space:]]*(-[[:space:]]*)?[\"']?environment[\"']?[[:space:]]*:[[:space:]]*[\"']?azureml:"
Expand Down
123 changes: 123 additions & 0 deletions tests/e2e/_aml.py
Original file line number Diff line number Diff line change
Expand Up @@ -574,6 +574,129 @@ def submit_aml_vla_pi0_training(
return _aml_job_from_submission(result, aml_workspace, experiment_name, "AzureML VLA pi0 training")


def register_aml_uri_folder_data_asset(
repo_root: Path,
aml_workspace: AzureMLWorkspace,
*,
name: str,
version: str,
local_path: Path,
) -> str:
"""Upload a local folder as a versioned URI-folder data asset and return its ``azureml:`` reference."""
log_e2e(f"Registering AzureML data asset {name}:{version} from {local_path}")
result = run_command(
[
"az",
"ml",
"data",
"create",
"--name",
name,
"--version",
version,
"--type",
"uri_folder",
"--path",
str(local_path),
*aml_workspace_args(aml_workspace),
"-o",
"json",
],
cwd=repo_root,
)
if result.returncode != 0:
raise AssertionError(
f"Failed to register AzureML data asset {name}:{version}\n\n{format_command_failure(result)}"
)
return f"azureml:{name}:{version}"


def submit_aml_groot_n17_training(
repo_root: Path,
aml_workspace: AzureMLWorkspace,
*,
dataset_asset: str,
modality_config: Path,
hf_token_secret_url: str,
num_gpus: int,
batch_size: int,
max_steps: int,
save_steps: int,
preflight_only: bool = False,
resume_checkpoint: str | None = None,
register_model_name: str | None = None,
) -> AzureMLJob:
phase = "preflight" if preflight_only else "resume" if resume_checkpoint else "train"
experiment_name = e2e_name(f"groot-n17-{phase}-e2e-aml")
instance_type = os.environ.get("E2E_AML_INSTANCE_TYPE", "")
log_e2e(
f"Submitting AzureML GR00T N1.7 {phase} job for dataset={dataset_asset}, num_gpus={num_gpus}, "
f"batch_size={batch_size}, max_steps={max_steps}, save_steps={save_steps}, "
f"resume_checkpoint={resume_checkpoint or '<none>'}, register_model={register_model_name or '<none>'}, "
f"experiment={experiment_name}, instance_type={instance_type or '<managed-compute>'}"
)
args = [
str(repo_root / "training/vla/scripts/submit-azureml-groot-training.sh"),
"--dataset-asset",
dataset_asset,
"--modality-config",
str(modality_config),
# Clear any ambient HF_TOKEN so the job resolves the token from Key Vault only.
"--hf-token",
"",
"--hf-token-secret-url",
hf_token_secret_url,
"--num-gpus",
str(num_gpus),
"--batch-size",
str(batch_size),
"--max-steps",
str(max_steps),
"--save-steps",
str(save_steps),
"--dataloader-workers",
"1",
"--experiment-name",
experiment_name,
*_submit_workspace_args(aml_workspace),
]
if instance_type:
args += ["--instance-type", instance_type]
if preflight_only:
args.append("--preflight-only")
if resume_checkpoint:
args += ["--resume-checkpoint", resume_checkpoint]
if register_model_name:
args += ["--register-model", register_model_name]

result = run_command(args, cwd=repo_root)
if result.returncode != 0:
raise AssertionError(f"AzureML GR00T N1.7 {phase} submission failed\n\n{format_command_failure(result)}")

return _aml_job_from_submission(result, aml_workspace, experiment_name, f"AzureML GR00T N1.7 {phase}")


def aml_job_output_datastore_uri(job: AzureMLJob, repo_root: Path, output_name: str) -> str:
"""Resolve a job's named output to a datastore URI accepted as a command-job input path."""
outputs = fetch_aml_job_payload(job, repo_root).get("outputs")
output = outputs.get(output_name) if isinstance(outputs, Mapping) else None
path = output.get("path") if isinstance(output, Mapping) else None
if isinstance(path, str) and path.startswith("azureml://") and "/datastores/" in path:
return path
# Outputs without an explicit path upload to the workspace default datastore location.
return f"azureml://datastores/workspaceblobstore/paths/azureml/{job.name}/{output_name}/"


def assert_aml_job_logs_contain(job: AzureMLJob, repo_root: Path, markers: Sequence[str]) -> str:
"""Download the job's user logs and assert that every marker appears."""
logs = fetch_aml_job_logs(job, repo_root)
missing = [marker for marker in markers if marker not in logs]
if missing:
raise AssertionError(f"AzureML job {job.name!r} logs did not contain {missing}")
log_e2e(f"AzureML job {job.name} logs contained {list(markers)}")
return logs


_AML_LEROBOT_EVAL_MODEL_ENV = "E2E_AML_LEROBOT_EVAL_MODEL"
_AML_ISAAC_EVAL_MODEL_ENV = "E2E_AML_ISAAC_EVAL_MODEL"

Expand Down
54 changes: 47 additions & 7 deletions tests/e2e/_vla_dataset.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,17 @@
"""
Synthetic GR00T (NVIDIA Isaac-GR00T N1.5) fine-tuning dataset generator.
Synthetic GR00T (NVIDIA Isaac-GR00T N1.5 and N1.7) fine-tuning dataset generator.

Produces a minimal, self-contained LeRobot v2.0 dataset plus the GR00T-specific
``meta/modality.json`` so the OSMO VLA fine-tuning e2e test no longer depends on a
manually pre-staged dataset. A single embodiment spec drives both the on-disk
dataset and the injected ``--data-config`` Python fragment, so the two cannot drift.
``meta/modality.json`` so the VLA fine-tuning e2e tests no longer depend on a
manually pre-staged dataset. A single embodiment spec drives the on-disk dataset,
the injected N1.5 ``--data-config`` Python fragment, and the N1.7 modality config,
so they cannot drift.

Schema is pinned to Isaac-GR00T ``796ca8d`` (the N1.5 ref used by
``training/vla/scripts/submit-osmo-lerobot-vla-fine-tuning.sh``):
GR00T ships its own reader (``gr00t/data/dataset.py``), so the ``lerobot`` pip
package is not required to load this dataset.
``training/vla/scripts/submit-osmo-lerobot-vla-fine-tuning.sh``) and is also
loadable by the N1.7 LeRobot episode loader at ``23ace64``.
GR00T ships its own reader, so the ``lerobot`` pip package is not required to
load this dataset.
"""

from __future__ import annotations
Expand Down Expand Up @@ -237,6 +239,44 @@ def write_data_config_file(path: Path) -> Path:
return path


def render_n17_modality_config_py() -> str:
"""GR00T N1.7 modality config matching ``meta/modality.json``.

Loaded by ``gr00t/experiment/launch_finetune.py`` via ``--modality_config_path``
and registered under ``EmbodimentTag.NEW_EMBODIMENT``.
"""
state_keys = [name for name, _ in _STATE_FIELDS]
action_keys = [name for name, _ in _ACTION_FIELDS]
return (
"from __future__ import annotations\n"
"\n"
"from gr00t.configs.data.embodiment_configs import register_modality_config\n"
"from gr00t.data.embodiment_tags import EmbodimentTag\n"
"from gr00t.data.types import ModalityConfig\n"
"\n"
"register_modality_config(\n"
" {\n"
f' "video": ModalityConfig(delta_indices=[0], modality_keys={json.dumps([_VIDEO_KEY])}),\n'
f' "state": ModalityConfig(delta_indices=[0], modality_keys={json.dumps(state_keys)}),\n'
' "action": ModalityConfig(\n'
f" delta_indices=list(range({_ACTION_HORIZON})),\n"
f" modality_keys={json.dumps(action_keys)},\n"
" ),\n"
' "language": ModalityConfig(\n'
" delta_indices=[0],\n"
f' modality_keys=["annotation.{_ANNOTATION_KEY}"],\n'
" ),\n"
" },\n"
" EmbodimentTag.NEW_EMBODIMENT,\n"
")\n"
)


def write_n17_modality_config_file(path: Path) -> Path:
path.write_text(render_n17_modality_config_py())
return path


def build_synthetic_dataset(root: Path) -> Path:
"""Write a complete GR00T-loadable dataset under ``root`` and return it."""
meta = root / "meta"
Expand Down
172 changes: 172 additions & 0 deletions tests/e2e/test_e2e_aml_groot_n17_training.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
"""
End-to-end lifecycle test for Azure ML GR00T N1.7 fine-tuning.

Registers a synthetic GR00T dataset as a versioned Azure ML data asset, then runs
four checks through ``training/vla/scripts/submit-azureml-groot-training.sh``:

1. A preflight job that resolves the Hugging Face token from Key Vault through the
Azure ML job identity and validates GPUs, dataset metadata, and the modality config.
2. A short training job that writes checkpoints to the ``checkpoints`` output.
3. A resume job that restores the latest checkpoint from the training job output.
4. Model registration from the resume job.

The test logs every Azure ML job name, terminal state, data asset, and model version
so the run can be cited as evidence.

Required environment:

* ``E2E_GROOT_HF_TOKEN_SECRET_URL``: Key Vault secret URL holding a Hugging Face token
with access to the gated ``nvidia/Cosmos-Reason2-2B`` backbone. The compute identity
needs ``Key Vault Secrets User`` on the secret.

Optional environment:

* ``E2E_AML_INSTANCE_TYPE``: InstanceType for Azure ML Kubernetes compute.
* ``E2E_GROOT_NUM_GPUS``: GPUs per job (default ``1``); must match the InstanceType.
* ``AZUREML_COMPUTE``: compute target override.

```shell
uv run pytest -vv -s -m e2e tests/e2e/test_e2e_aml_groot_n17_training.py
```
"""

from __future__ import annotations

import os
import shutil
import tempfile
from pathlib import Path

import pytest

from tests.e2e._aml import (
AzureMLJob,
AzureMLWorkspace,
aml_job_output_datastore_uri,
archive_aml_data_asset,
assert_aml_job_logs_contain,
assert_job_has_checkpoint,
assert_registered_model_has_artifacts,
cancel_aml_job,
cleanup_aml_job_and_model_versions,
register_aml_uri_folder_data_asset,
resolve_registered_model,
submit_aml_groot_n17_training,
wait_until_aml_completed,
wait_until_aml_started,
)
from tests.e2e._common import e2e_name, log_e2e, parse_provenance_marker

_HF_TOKEN_SECRET_URL_ENV = "E2E_GROOT_HF_TOKEN_SECRET_URL"
_NUM_GPUS_ENV = "E2E_GROOT_NUM_GPUS"
_KEY_VAULT_MARKER = "[azureml] loaded Hugging Face token from Key Vault"
_PROVENANCE_MARKER = "VLA_RUNTIME_PROVENANCE="
_TRAIN_STEPS = 2
_RESUME_STEPS = 4
# Kubernetes GPU nodes scale from zero and the default image installs GR00T at job start.
_START_TIMEOUT_MINUTES = 40
_COMPLETE_TIMEOUT_MINUTES = 90


def _run_to_completion(job: AzureMLJob, repo_root: Path, phase: str) -> None:
log_e2e(f"Waiting for AzureML GR00T N1.7 {phase} job {job.name} to start")
wait_until_aml_started(job, repo_root, timeout_minutes=_START_TIMEOUT_MINUTES, poll_interval_seconds=30)
log_e2e(f"Waiting for AzureML GR00T N1.7 {phase} job {job.name} to complete")
wait_until_aml_completed(job, repo_root, timeout_minutes=_COMPLETE_TIMEOUT_MINUTES, poll_interval_seconds=30)


@pytest.mark.e2e
@pytest.mark.usefixtures("aml_compute_target")
def test_aml_groot_n17_lifecycle_e2e(
request: pytest.FixtureRequest,
aml_workspace: AzureMLWorkspace,
repo_root: Path,
) -> None:
hf_token_secret_url = os.environ.get(_HF_TOKEN_SECRET_URL_ENV, "").strip()
if not hf_token_secret_url:
pytest.skip(f"{_HF_TOKEN_SECRET_URL_ENV} is not set")
num_gpus = int(os.environ.get(_NUM_GPUS_ENV, "1"))

# Imported lazily so the numpy/pyarrow/imageio generator stack only loads when the test runs.
from tests.e2e._vla_dataset import (
build_synthetic_dataset,
validate_synthetic_dataset,
write_n17_modality_config_file,
)

log_e2e("Starting AzureML GR00T N1.7 lifecycle e2e test")
work_dir = Path(tempfile.mkdtemp(prefix="groot-n17-e2e-"))
request.addfinalizer(lambda: shutil.rmtree(work_dir, ignore_errors=True))
dataset_dir = build_synthetic_dataset(work_dir / "dataset")
validate_synthetic_dataset(dataset_dir)
modality_config = write_n17_modality_config_file(work_dir / "e2e_synth_modality_config.py")

dataset_name = e2e_name("groot-n17-e2e-dataset")
request.addfinalizer(lambda: archive_aml_data_asset(repo_root, aml_workspace, dataset_name))
dataset_asset = register_aml_uri_folder_data_asset(
repo_root, aml_workspace, name=dataset_name, version="1", local_path=dataset_dir
)

common = {
"dataset_asset": dataset_asset,
"modality_config": modality_config,
"hf_token_secret_url": hf_token_secret_url,
"num_gpus": num_gpus,
"batch_size": num_gpus,
"save_steps": 1,
}

preflight_job = submit_aml_groot_n17_training(
repo_root, aml_workspace, max_steps=_TRAIN_STEPS, preflight_only=True, **common
)
request.addfinalizer(lambda: cancel_aml_job(preflight_job, repo_root))
_run_to_completion(preflight_job, repo_root, "preflight")
assert_aml_job_logs_contain(preflight_job, repo_root, [_KEY_VAULT_MARKER, "TRAIN_RESULT=PREFLIGHT_PASS"])

train_job = submit_aml_groot_n17_training(repo_root, aml_workspace, max_steps=_TRAIN_STEPS, **common)
request.addfinalizer(lambda: cancel_aml_job(train_job, repo_root))
_run_to_completion(train_job, repo_root, "training")
train_logs = assert_aml_job_logs_contain(train_job, repo_root, [_KEY_VAULT_MARKER])
train_provenance = parse_provenance_marker(train_logs, _PROVENANCE_MARKER)
assert train_provenance["resume_from_checkpoint"] is False
assert_job_has_checkpoint(train_job)

register_model_name = e2e_name("groot-n17-e2e-aml-model")
resume_job = submit_aml_groot_n17_training(
repo_root,
aml_workspace,
max_steps=_RESUME_STEPS,
resume_checkpoint=aml_job_output_datastore_uri(train_job, repo_root, "checkpoints"),
register_model_name=register_model_name,
**common,
)
request.addfinalizer(
lambda: cleanup_aml_job_and_model_versions(resume_job, repo_root, aml_workspace, register_model_name)
)
_run_to_completion(resume_job, repo_root, "resume")
resume_logs = assert_aml_job_logs_contain(
resume_job,
repo_root,
[_KEY_VAULT_MARKER, "[setup] patched N1.7 Trainer for deterministic data resume"],
)
linked = [line for line in resume_logs.splitlines() if "[azureml] linked resume checkpoint: " in line]
assert linked, "resume job did not link a checkpoint from the training job output"
assert linked[-1].rstrip().endswith(f"checkpoint-{_TRAIN_STEPS}"), linked[-1]
resume_provenance = parse_provenance_marker(resume_logs, _PROVENANCE_MARKER)
assert resume_provenance["resume_from_checkpoint"] is True
assert resume_provenance["isaac_groot_ref"] == train_provenance["isaac_groot_ref"]
assert_job_has_checkpoint(resume_job)

model = resolve_registered_model(repo_root, aml_workspace, model_name=register_model_name)
assert_registered_model_has_artifacts(repo_root, aml_workspace, model)

log_e2e(
"AzureML GR00T N1.7 lifecycle evidence: "
f"dataset={dataset_asset}; "
f"preflight={preflight_job.name}:{preflight_job.terminal_status}; "
f"train={train_job.name}:{train_job.terminal_status}; "
f"resume={resume_job.name}:{resume_job.terminal_status}; "
f"model={model.name}:{model.version}; "
f"isaac_groot_ref={train_provenance['isaac_groot_ref']}"
)
log_e2e("AzureML GR00T N1.7 lifecycle e2e test finished successfully")
Loading
Loading