environments ai ml automl dnn text gpu ptca - Azure/azureml-assets GitHub Wiki

ai-ml-automl-dnn-text-gpu-ptca

Overview

An environment used by Azure ML AutoML for training models.

Version: 56

Tags

OS : Ubuntu20.04 Training Preview OpenMpi : 4.1.0 Python : 3.9

View in Studio: https://ml.azure.com/registries/azureml/environments/ai-ml-automl-dnn-text-gpu-ptca/version/56

Docker image: mcr.microsoft.com/azureml/curated/ai-ml-automl-dnn-text-gpu-ptca:56

Docker build context

Dockerfile

FROM mcr.microsoft.com/aifx/acpt/stable-ubuntu2204-cu126-py310-torch280:biweekly.202610.1

USER root:root

# Upgrade Ubuntu security fixes inherited from the ACPT base image.
RUN apt-get update && \
    apt-get upgrade -y && \
    apt-get clean && rm -rf /var/lib/apt/lists/* && \
    apt-get autoremove -y

RUN pip install --no-cache-dir --use-deprecated=legacy-resolver \
    'azureml-automl-dnn-nlp==1.62.0' \
    'azureml-defaults==1.62.0'

# onnx and onnxruntime-training installation
RUN pip uninstall -y onnxruntime
RUN pip uninstall -y onnxruntime-training
RUN pip install -i https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/ onnxruntime-training==1.18.0

# torch-ort installation
RUN TORCH_CUDA_ARCH_LIST="5.2;6.0;7.0;8.0;8.6;9.0" python -m onnxruntime.training.ortmodule.torch_cpp_extensions.install
RUN pip install torch-ort==1.18.0 && TORCH_CUDA_ARCH_LIST="5.2;6.0;7.0;8.0;8.6;9.0" python -m torch_ort.configure
RUN pip uninstall -y onnxruntime

RUN pip install \
    optimum==1.23.3 \
    accelerate==1.12.0 \
    deepspeed~=0.15.1

# Override transformers to fix GHSA-69w3-r845-3855 and GHSA-xrqw-3rrv-vx5w.
# Root cause: azureml-automl-dnn-nlp pins transformers below the patched
# version, so direct override is required.
RUN pip install --no-cache-dir --no-deps 'transformers[sentencepiece,torch]==5.10.0'

# Vulnerability patches for ptca environment (python 3.10 at /opt/conda/envs/ptca)
# pip -> >=26.1.2 (GHSA-jp4c-xjxw-mgf9, CVE-2026-6357,
#   GHSA-wf93-45jw-7689 / CVE-2026-8643): no parent
#   package controls pip here, so direct upgrade is required. The stale
#   conda-meta JSON files for older pip builds are removed so scanners do not
#   re-flag them.
# setuptools 81.0.0 -> >=83.0.0 (GHSA-h35f-9h28-mq5c, GHSA-58pv-8j8x-9vj2):
#   setuptools is a build dependency with no parent that pins it.
# numpy<2 is NOT a security floor: onnx pulls ml_dtypes, which as of 0.6.0 requires
#   numpy>=2.0.0. Without this cap pip silently upgrades numpy to 2.x, leaving the
#   prebuilt pandas wheel binary-incompatible so the image fails on import pandas with
#   "ValueError: numpy.dtype size changed".
# Override onnx to fix GHSA-cmw6-hcpp-c6jp, GHSA-538c-55jv-c5g9, GHSA-q56x-g2fj-4rj6,
#   GHSA-p433-9wv8-28xj, GHSA-3r9x-f23j-gc73, GHSA-hqmj-h5c6-369m.
#   Root cause: azureml-automl-runtime==1.62.0 is the latest release as of
#   2026-05-26 and pins onnx<=1.17.0, so direct override is required.
# bokeh 2.4.3 -> >=3.8.2 (GHSA-793v-589g-574v, CVE-2026-21883) and distributed
#   2023.2.0 -> >=2026.1.0 (GHSA-c336-7962-wfj2, CVE-2026-23528) are pulled in by
#   azureml-train-automl-runtime==1.62.0, which requires bokeh<3.0.0 and
#   dask[complete]<=2023.2.0. Latest AzureML parents remain at 1.62.0 as of
#   2026-05-26, so direct override is required.
# aiohttp 3.13.5 -> >=3.14.0 (GHSA-jg22-mg44-37j8, GHSA-hg6j-4rv6-33pg): aiohttp
#   is pulled in transitively with no parent pinning it to <3.14.0, so direct
#   override is required.
# pyarrow 17.0.0 -> >=23.0.1 (GHSA-rgxp-2hwp-jwgg / CVE-2026-25087): pinned
#   transitive dep of azureml-automl-dnn-nlp via AzureML dataset/runtime packages.
# cryptography -> >=50.0.0 (GHSA-g6cj-pr64-35w5): pinned transitive dep
#   brought in by the ACPT base image ptca conda environment.
# GitPython -> >=3.1.57 (GHSA-3f7w-8rr8-f37f, GHSA-539m-9xh6-q6rr,
#   GHSA-p538-c434-8v24): pinned transitive dep of AzureML/MLflow.
# sqlparse -> >=0.6.0 (GHSA-f2ff-p2ww-7p4p, GHSA-prg7-hcfm-mfcr, GHSA-3496-9g83-7v6x,
#   GHSA-pwgv-4x5q-6m9f): pinned transitive dep of MLflow; no parent release floors
#   the patched version, so direct override is required.
RUN conda install -p /opt/conda/envs/ptca -c conda-forge -y 'pip>=26.1.2' && \
    /opt/conda/envs/ptca/bin/pip install --no-cache-dir --upgrade \
    'setuptools>=83.0.0' \
    'msgpack>=1.2.1' \
    'numpy<2' \
    'onnx>=1.21.0' \
    'bokeh>=3.8.2' \
    'distributed>=2026.1.0' \
    'aiohttp>=3.14.0' \
    'httpx<1' \
    'huggingface-hub>=1.5.0,<2.0' \
    'pyarrow>=23.0.1' \
    'cryptography>=50.0.0' \
    'GitPython>=3.1.57' \
    'sqlparse>=0.6.0' && \
    rm -f /opt/conda/envs/ptca/conda-meta/pip-26.0.1-*.json \
          /opt/conda/envs/ptca/conda-meta/pip-26.1.1-*.json

# Vulnerability patches for the conda base environment at /opt/conda.
# pip 26.0.1 -> >=26.1.1 (GHSA-jp4c-xjxw-mgf9, CVE-2026-6357): no parent
#   package controls pip here, so direct upgrade is required.
# urllib3 2.6.3 -> >=2.7.0 (GHSA-qccp-gfcp-xxvc / CVE-2026-44431, GHSA-mf9v-mfxr-j63j
#   / CVE-2026-44432): requests 2.34.2 is the latest release as of 2026-05-26 and
#   still allows urllib3<3,>=1.26; conda CLI parents also use loose urllib3 ranges,
#   so no parent upgrade can force >=2.7.0 and direct override is required.
# idna >=3.15 (GHSA-65pc-fj4g-8rjx): parents (`requests`, `anyio`, `httpx`,
#   `yarl`) declare loose idna requirements, so direct override is required.
# click >=8.3.3 (GHSA-47fr-3ffg-hgmw): CLI parents use loose click floors, so
#   direct override is required.
# setuptools 82.0.0 -> >=83.0.0 (GHSA-h35f-9h28-mq5c, GHSA-58pv-8j8x-9vj2):
#   no parent package pins setuptools.
# python-dotenv 1.2.1 -> >=1.2.2 (GHSA-mf9w-mj56-hr94): brought in transitively by
#   anaconda-auth==0.14.4 (Requires-Dist: python-dotenv with no version pin) and
#   pydantic-settings==2.12.0 (python-dotenv>=0.21.0, via anaconda-cli-base ->
#   anaconda-auth). Latest releases on PyPI as of 2026-05-26 (anaconda-auth==0.15.0,
#   pydantic-settings==2.14.1) still use the same loose floors, so a parent upgrade
#   cannot force >=1.2.2 -- direct override required.
# aiohttp 3.13.5 -> >=3.14.0 (GHSA-jg22-mg44-37j8, GHSA-hg6j-4rv6-33pg): aiohttp
#   is pulled in transitively with no parent pinning it to <3.14.0, so direct
#   override is required.
# PyJWT 2.12.1 -> >=2.13.0 (GHSA-jq35-7prp-9v3f, GHSA-993g-76c3-p5m4): pinned
#   transitive dep brought in by the ACPT base image conda environment.
# py-rattler 0.23.1 -> >=0.24.0 (GHSA-q53q-5r4j-5729): pinned transitive dep
#   brought in by the ACPT base image conda tooling.
# cryptography -> >=50.0.0 (GHSA-g6cj-pr64-35w5): pinned transitive dep
#   brought in by the ACPT base image conda environment.
# pydantic-settings 2.12.0 -> >=2.14.2 (GHSA-4xgf-cpjx-pc3j): transitive dep
#   brought in by the ACPT base image conda environment.
# msgpack 1.1.1 -> >=1.2.1 (GHSA-6v7p-g79w-8964): transitive dep brought in by
#   mlflow-skinny/AzureML dependencies in the base Python environment.
# Stale conda pkgs-cache entries for click, idna, pip, and python-dotenv are removed
#   so scanners do not re-flag the old versions from the package cache directories.
RUN /opt/conda/bin/pip install --no-cache-dir --upgrade \
    'pip>=26.1.1' \
    'urllib3>=2.7.0' \
    'idna>=3.15' \
    'click>=8.3.3' \
    'setuptools>=83.0.0' \
    'python-dotenv>=1.2.2' \
    'aiohttp>=3.14.0' \
    'anyio>=4.14.2' \
    'PyJWT>=2.13.0' \
    'py-rattler>=0.24.0' \
    'cryptography>=50.0.0' \
    'pydantic-settings>=2.14.2' \
    'msgpack>=1.2.1' && \
    rm -rf /opt/conda/pkgs/click-8.2.1-* \
           /opt/conda/pkgs/idna-3.11-* \
           /opt/conda/pkgs/pip-26.0.1-* \
           /opt/conda/pkgs/python-dotenv-1.2.1-* \
           /opt/conda/pkgs/msgpack-1.1.2-* \
           /opt/conda/pkgs/setuptools-70.3.0-*

# Temporary override for the Torch stack inherited from the ACPT torch280 base.
# The CUDA 12.6 index has no torchaudio build compatible with torch>=2.12, so
# remove inherited torchaudio rather than retaining a binary-incompatible build.
# Remove when the base provides torch>=2.13.0 and compatible companion packages.
RUN /opt/conda/envs/ptca/bin/pip uninstall -y torchaudio && \
    /opt/conda/envs/ptca/bin/pip install --no-cache-dir --upgrade \
        --index-url https://download.pytorch.org/whl/cu126 \
        'torch==2.13.0' 'torchvision==0.28.0' && \
    /opt/conda/envs/ptca/bin/pip install --no-cache-dir --upgrade --no-deps \
        'setuptools>=83.0.0' 'huggingface-hub==1.33.0' 'tokenizers==0.22.2'

# pip 26.2.1's vendor inventories report its private setuptools 70.3.0 and
# msgpack 1.1.2 copies as pathless packages. Remove generated pip vendor
# CycloneDX inventories and matching stale entries, then fail if any remain.
RUN set -eu; \
    set -- /opt/conda; \
    if test -d /azureml-envs; then set -- "$@" /azureml-envs; fi; \
    find "$@" -type f \
        -path '*/site-packages/pip/_vendor/bom.cdx.json' -delete; \
    find "$@" -type f -path '*/pip/_vendor/vendor.txt' \
        -exec sed -i '/^setuptools==70\.3\.0$/d; /^msgpack==1\.1\.2$/d' {} +; \
    for python in /opt/conda/bin/python /opt/conda/envs/*/bin/python /azureml-envs/*/bin/python; do \
        test -x "$python" || continue; \
        site_packages="$("$python" -c 'import sysconfig; print(sysconfig.get_path("purelib"))')"; \
        find "$site_packages" -type f \( \
            -path '*/pip-*.dist-info/sboms/bom.cdx.json' -o \
            -path '*/virtualenv-*.dist-info/sboms/*.json' \
        \) -exec grep -IlE 'pkg:pypi/(setuptools@70\.3\.0|msgpack@1\.1\.2)(["?#]|$)' {} + | \
            while IFS= read -r inventory; do rm -f "$inventory"; done; \
        if find "$site_packages" -type f \( \
            -path '*/pip-*.dist-info/sboms/bom.cdx.json' -o \
            -path '*/virtualenv-*.dist-info/sboms/*.json' \
        \) -exec grep -IlE 'pkg:pypi/(setuptools@70\.3\.0|msgpack@1\.1\.2)(["?#]|$)' {} + | grep -q .; then exit 1; fi; \
    done; \
    if find "$@" -type f \
        -path '*/site-packages/pip/_vendor/bom.cdx.json' | grep -q .; then exit 1; fi; \
    if find "$@" -type f -path '*/pip/_vendor/vendor.txt' \
        -exec grep -IlE '^(setuptools==70\.3\.0|msgpack==1\.1\.2)$' {} + | grep -q .; then exit 1; fi
⚠️ **GitHub.com Fallback** ⚠️