Skip to content
Open
Show file tree
Hide file tree
Changes from 13 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions scripts/update-image-digests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ azureml_pin_specs=(
'DEFAULT_LEROBOT_EVAL_IMAGE|lerobot-inference-env|evaluation/sil/workflows/azureml/lerobot-eval.yaml'
'DEFAULT_LEROBOT_EVAL_IMAGE|lerobot-inference-env|evaluation/sil/workflows/azureml/vla-pi0-eval.yaml'
'DEFAULT_LEROBOT_TRAIN_IMAGE|vla-pi0-training-env|training/vla/workflows/azureml/vla-pi0-train.yaml'
'DEFAULT_GROOT_IMAGE|groot-n17-training-env|training/vla/workflows/azureml/groot-n17-train.yaml'
'DEFAULT_AZUREML_SMOKE_IMAGE|gpu-smoke-env|training/smoke/workflows/azureml/gpu-smoke.yaml'
)
azureml_environment_line_re="^[[:space:]]*(-[[:space:]]*)?[\"']?environment[\"']?[[:space:]]*:[[:space:]]*[\"']?azureml:"
Expand Down
189 changes: 189 additions & 0 deletions tests/e2e/_aml.py
Original file line number Diff line number Diff line change
Expand Up @@ -574,6 +574,195 @@ def submit_aml_vla_pi0_training(
return _aml_job_from_submission(result, aml_workspace, experiment_name, "AzureML VLA pi0 training")


def register_aml_uri_folder_data_asset(
repo_root: Path,
aml_workspace: AzureMLWorkspace,
*,
name: str,
version: str,
local_path: Path,
) -> str:
"""Upload a local folder as a versioned URI-folder data asset and return its ``azureml:`` reference."""
log_e2e(f"Registering AzureML data asset {name}:{version} from {local_path}")
result = run_command(
[
"az",
"ml",
"data",
"create",
"--name",
name,
"--version",
version,
"--type",
"uri_folder",
"--path",
str(local_path),
*aml_workspace_args(aml_workspace),
"-o",
"json",
],
cwd=repo_root,
)
if result.returncode != 0:
raise AssertionError(
f"Failed to register AzureML data asset {name}:{version}\n\n{format_command_failure(result)}"
)
return f"azureml:{name}:{version}"


def submit_aml_groot_n17_training(
repo_root: Path,
aml_workspace: AzureMLWorkspace,
*,
dataset_asset: str,
modality_config: Path,
hf_token_secret_url: str,
num_gpus: int,
batch_size: int,
max_steps: int,
save_steps: int,
preflight_only: bool = False,
resume_checkpoint: str | None = None,
register_model_name: str | None = None,
) -> AzureMLJob:
phase = "preflight" if preflight_only else "resume" if resume_checkpoint else "train"
experiment_name = e2e_name(f"groot-n17-{phase}-e2e-aml")
instance_type = os.environ.get("E2E_AML_INSTANCE_TYPE", "")
log_e2e(
f"Submitting AzureML GR00T N1.7 {phase} job for dataset={dataset_asset}, num_gpus={num_gpus}, "
f"batch_size={batch_size}, max_steps={max_steps}, save_steps={save_steps}, "
f"resume_checkpoint={resume_checkpoint or '<none>'}, register_model={register_model_name or '<none>'}, "
f"experiment={experiment_name}, instance_type={instance_type or '<managed-compute>'}"
)
args = [
str(repo_root / "training/vla/scripts/submit-azureml-groot-training.sh"),
"--dataset-asset",
dataset_asset,
"--modality-config",
str(modality_config),
# Clear any ambient HF_TOKEN so the job resolves the token from Key Vault only.
"--hf-token",
"",
"--hf-token-secret-url",
hf_token_secret_url,
"--num-gpus",
str(num_gpus),
"--batch-size",
str(batch_size),
"--max-steps",
str(max_steps),
"--save-steps",
str(save_steps),
"--dataloader-workers",
"1",
"--experiment-name",
experiment_name,
*_submit_workspace_args(aml_workspace),
]
if instance_type:
args += ["--instance-type", instance_type]
if preflight_only:
args.append("--preflight-only")
if resume_checkpoint:
args += ["--resume-checkpoint", resume_checkpoint]
if register_model_name:
args += ["--register-model", register_model_name]

result = run_command(args, cwd=repo_root)
if result.returncode != 0:
raise AssertionError(f"AzureML GR00T N1.7 {phase} submission failed\n\n{format_command_failure(result)}")

return _aml_job_from_submission(result, aml_workspace, experiment_name, f"AzureML GR00T N1.7 {phase}")


def aml_job_output_datastore_uri(job: AzureMLJob, repo_root: Path, output_name: str) -> str:
"""Resolve a job's named output to a datastore URI accepted as a command-job input path."""
outputs = fetch_aml_job_payload(job, repo_root).get("outputs")
output = outputs.get(output_name) if isinstance(outputs, Mapping) else None
path = output.get("path") if isinstance(output, Mapping) else None
if isinstance(path, str) and path.startswith("azureml://") and "/datastores/" in path:
return path
# Outputs without an explicit path upload to the workspace default datastore location.
return f"azureml://datastores/workspaceblobstore/paths/azureml/{job.name}/{output_name}/"


def assert_aml_job_logs_contain(job: AzureMLJob, repo_root: Path, markers: Sequence[str]) -> str:
"""Download the job's user logs and assert that every marker appears."""
logs = fetch_aml_job_logs(job, repo_root)
missing = [marker for marker in markers if marker not in logs]
if missing:
raise AssertionError(f"AzureML job {job.name!r} logs did not contain {missing}")
log_e2e(f"AzureML job {job.name} logs contained {list(markers)}")
return logs


_DATASTORE_PATH_RE = re.compile(r"/datastores/(?P<datastore>[^/]+)/paths/(?P<prefix>.+)$")


def assert_registered_model_has_blob_artifacts(
repo_root: Path,
aml_workspace: AzureMLWorkspace,
model: AmlModelRef,
) -> None:
"""List a registered model's artifact blobs without downloading multi-GB weights."""
from azure.ai.ml import MLClient
from azure.identity import DefaultAzureCredential
from azure.storage.blob import ContainerClient

result = run_command(
[
"az",
"ml",
"model",
"show",
"--name",
model.name,
"--version",
model.version,
*aml_workspace_args(aml_workspace),
"-o",
"json",
],
cwd=repo_root,
)
if result.returncode != 0:
raise AssertionError(
f"Unable to show AzureML model {model.name!r}:{model.version}\n\n{format_command_failure(result)}"
)
payload = parse_json_from_output(result.stdout)
path = payload.get("path") if isinstance(payload, Mapping) else None
match = _DATASTORE_PATH_RE.search(path) if isinstance(path, str) else None
if match is None:
raise AssertionError(f"AzureML model {model.name!r}:{model.version} has no datastore path: {path!r}")

credential = DefaultAzureCredential()
client = MLClient(
credential=credential,
subscription_id=aml_workspace.subscription_id,
resource_group_name=aml_workspace.resource_group,
workspace_name=aml_workspace.workspace_name,
)
datastore = client.datastores.get(match.group("datastore"))
container = ContainerClient(
account_url=f"https://{datastore.account_name}.blob.core.windows.net",
container_name=datastore.container_name,
credential=credential,
)
prefix = match.group("prefix").rstrip("/") + "/"
with container:
blobs = list(container.list_blobs(name_starts_with=prefix))
if not blobs:
raise AssertionError(f"AzureML model {model.name!r}:{model.version} has no artifact files under {prefix}")

size_gib = sum(blob.size or 0 for blob in blobs) / 1024**3
sample = ", ".join(sorted(blob.name.removeprefix(prefix) for blob in blobs)[:5])
log_e2e(
f"AzureML model artifacts passed: model={model.name}:{model.version}, "
f"artifacts={len(blobs)}, size={size_gib:.2f} GiB; sample: {sample}"
)


_AML_LEROBOT_EVAL_MODEL_ENV = "E2E_AML_LEROBOT_EVAL_MODEL"
_AML_ISAAC_EVAL_MODEL_ENV = "E2E_AML_ISAAC_EVAL_MODEL"

Expand Down
54 changes: 47 additions & 7 deletions tests/e2e/_vla_dataset.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,17 @@
"""
Synthetic GR00T (NVIDIA Isaac-GR00T N1.5) fine-tuning dataset generator.
Synthetic GR00T (NVIDIA Isaac-GR00T N1.5 and N1.7) fine-tuning dataset generator.

Produces a minimal, self-contained LeRobot v2.0 dataset plus the GR00T-specific
``meta/modality.json`` so the OSMO VLA fine-tuning e2e test no longer depends on a
manually pre-staged dataset. A single embodiment spec drives both the on-disk
dataset and the injected ``--data-config`` Python fragment, so the two cannot drift.
``meta/modality.json`` so the VLA fine-tuning e2e tests no longer depend on a
manually pre-staged dataset. A single embodiment spec drives the on-disk dataset,
the injected N1.5 ``--data-config`` Python fragment, and the N1.7 modality config,
so they cannot drift.

Schema is pinned to Isaac-GR00T ``796ca8d`` (the N1.5 ref used by
``training/vla/scripts/submit-osmo-lerobot-vla-fine-tuning.sh``):
GR00T ships its own reader (``gr00t/data/dataset.py``), so the ``lerobot`` pip
package is not required to load this dataset.
``training/vla/scripts/submit-osmo-lerobot-vla-fine-tuning.sh``) and is also
loadable by the N1.7 LeRobot episode loader at ``23ace64``.
GR00T ships its own reader, so the ``lerobot`` pip package is not required to
load this dataset.
"""

from __future__ import annotations
Expand Down Expand Up @@ -237,6 +239,44 @@ def write_data_config_file(path: Path) -> Path:
return path


def render_n17_modality_config_py() -> str:
"""GR00T N1.7 modality config matching ``meta/modality.json``.

Loaded by ``gr00t/experiment/launch_finetune.py`` via ``--modality_config_path``
and registered under ``EmbodimentTag.NEW_EMBODIMENT``.
"""
state_keys = [name for name, _ in _STATE_FIELDS]
action_keys = [name for name, _ in _ACTION_FIELDS]
return (
"from __future__ import annotations\n"
"\n"
"from gr00t.configs.data.embodiment_configs import register_modality_config\n"
"from gr00t.data.embodiment_tags import EmbodimentTag\n"
"from gr00t.data.types import ModalityConfig\n"
"\n"
"register_modality_config(\n"
" {\n"
f' "video": ModalityConfig(delta_indices=[0], modality_keys={json.dumps([_VIDEO_KEY])}),\n'
f' "state": ModalityConfig(delta_indices=[0], modality_keys={json.dumps(state_keys)}),\n'
' "action": ModalityConfig(\n'
f" delta_indices=list(range({_ACTION_HORIZON})),\n"
f" modality_keys={json.dumps(action_keys)},\n"
" ),\n"
' "language": ModalityConfig(\n'
" delta_indices=[0],\n"
f' modality_keys=["annotation.{_ANNOTATION_KEY}"],\n'
" ),\n"
" },\n"
" EmbodimentTag.NEW_EMBODIMENT,\n"
")\n"
)


def write_n17_modality_config_file(path: Path) -> Path:
path.write_text(render_n17_modality_config_py())
return path


def build_synthetic_dataset(root: Path) -> Path:
"""Write a complete GR00T-loadable dataset under ``root`` and return it."""
meta = root / "meta"
Expand Down
Loading
Loading