From f7f957aacb7b8df827f6b079a43ea5aa125671f3 Mon Sep 17 00:00:00 2001 From: Jake Stevens Date: Tue, 21 Jul 2026 08:33:05 -0700 Subject: [PATCH 1/2] ci: mitigate OOM kills on large-model export tests Large models (e.g. phi4 ~14.7B) exceed the 16GB RAM on standard ubuntu-22.04 runners and get OOM-killed during model load/quantization, surfacing as exit code 143 / 'The operation was canceled' rather than a test failure. Add an aggressive disk cleanup step and a 24GB swapfile so these jobs page to disk instead of being killed, plus a timeout-minutes so genuine hangs fail cleanly instead of dragging the run for hours. Applied to both the stable (test_models.yml) and nightly (test_models_nightly.yml) workflows. --- .github/workflows/test_models.yml | 28 +++++++++++++++++++++++ .github/workflows/test_models_nightly.yml | 28 +++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/.github/workflows/test_models.yml b/.github/workflows/test_models.yml index 20f5474..3860f9c 100644 --- a/.github/workflows/test_models.yml +++ b/.github/workflows/test_models.yml @@ -42,6 +42,9 @@ jobs: # Custom job name, now shortened and cleaner name: ${{ matrix.test-modeling }} (et=${{ matrix.executorch-version }}, py=${{ matrix.python-version }}, ${{ matrix.os }}) runs-on: ${{ matrix.os }} + # Fail cleanly on a hang instead of dragging the run for hours; large-model + # export can be slow (especially once paging to swap), so keep this generous. + timeout-minutes: 180 env: MODEL_NAME: ${{ matrix.test-modeling }} steps: @@ -50,6 +53,31 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + - name: Free up disk space + run: | + echo "Disk space before cleanup:" + df -h / + # Reclaim space from preinstalled toolchains we don't use, so there's + # room for a large swapfile and downloaded model weights. + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/boost "$AGENT_TOOLSDIRECTORY" || true + sudo apt-get clean || true + echo "Disk space after cleanup:" + df -h / + - name: Increase swap space + run: | + # Large models (e.g. phi4 ~14.7B) exceed the 16GB RAM on standard + # runners and get OOM-killed during load/quantization (exit 143). A + # big swapfile lets them page to disk instead of being killed. + echo "Memory before:" + free -h + sudo swapoff -a || true + sudo rm -f /mnt/swapfile || true + sudo fallocate -l 24G /mnt/swapfile || sudo dd if=/dev/zero of=/mnt/swapfile bs=1M count=24576 + sudo chmod 600 /mnt/swapfile + sudo mkswap /mnt/swapfile + sudo swapon /mnt/swapfile + echo "Memory after:" + free -h - name: Install dependencies for ExecuTorch run: | # Clean up cache to save space diff --git a/.github/workflows/test_models_nightly.yml b/.github/workflows/test_models_nightly.yml index aabc021..137386c 100644 --- a/.github/workflows/test_models_nightly.yml +++ b/.github/workflows/test_models_nightly.yml @@ -38,6 +38,9 @@ jobs: name: ${{ matrix.test-modeling }} (et=nightly, py=${{ matrix.python-version }}, ${{ matrix.os }}) runs-on: ${{ matrix.os }} + # Fail cleanly on a hang instead of dragging the run for hours; large-model + # export can be slow (especially once paging to swap), so keep this generous. + timeout-minutes: 180 env: MODEL_NAME: ${{ matrix.test-modeling }} steps: @@ -46,6 +49,31 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + - name: Free up disk space + run: | + echo "Disk space before cleanup:" + df -h / + # Reclaim space from preinstalled toolchains we don't use, so there's + # room for a large swapfile and downloaded model weights. + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/boost "$AGENT_TOOLSDIRECTORY" || true + sudo apt-get clean || true + echo "Disk space after cleanup:" + df -h / + - name: Increase swap space + run: | + # Large models (e.g. phi4 ~14.7B) exceed the 16GB RAM on standard + # runners and get OOM-killed during load/quantization (exit 143). A + # big swapfile lets them page to disk instead of being killed. + echo "Memory before:" + free -h + sudo swapoff -a || true + sudo rm -f /mnt/swapfile || true + sudo fallocate -l 24G /mnt/swapfile || sudo dd if=/dev/zero of=/mnt/swapfile bs=1M count=24576 + sudo chmod 600 /mnt/swapfile + sudo mkswap /mnt/swapfile + sudo swapon /mnt/swapfile + echo "Memory after:" + free -h - name: Install dependencies for ExecuTorch nightly run: | pip cache purge || true From a205292d6af1b0f95cf1343c5c8aacf75eb56704 Mon Sep 17 00:00:00 2001 From: Jake Stevens Date: Tue, 21 Jul 2026 12:49:03 -0700 Subject: [PATCH 2/2] ci: drop disk-cleanup step that wiped the Python toolcache The 'Free up disk space' step deleted $AGENT_TOOLSDIRECTORY (/opt/hostedtoolcache), which is where actions/setup-python installs Python -- removing it broke pip/pytest and caused 'pytest: command not found' on every job. Root already has ~87GB free and the swapfile lives on the separate /mnt disk, so the cleanup was unnecessary. Remove it and keep only the swapfile + timeout. --- .github/workflows/test_models.yml | 16 +++------------- .github/workflows/test_models_nightly.yml | 16 +++------------- 2 files changed, 6 insertions(+), 26 deletions(-) diff --git a/.github/workflows/test_models.yml b/.github/workflows/test_models.yml index 3860f9c..3611f25 100644 --- a/.github/workflows/test_models.yml +++ b/.github/workflows/test_models.yml @@ -53,21 +53,11 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Free up disk space - run: | - echo "Disk space before cleanup:" - df -h / - # Reclaim space from preinstalled toolchains we don't use, so there's - # room for a large swapfile and downloaded model weights. - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/boost "$AGENT_TOOLSDIRECTORY" || true - sudo apt-get clean || true - echo "Disk space after cleanup:" - df -h / - name: Increase swap space run: | - # Large models (e.g. phi4 ~14.7B) exceed the 16GB RAM on standard - # runners and get OOM-killed during load/quantization (exit 143). A - # big swapfile lets them page to disk instead of being killed. + # Large models (e.g. gemma-3-4b, Phi-4-mini) push past the 16GB RAM on + # standard runners and get OOM-killed during load/quantization + # (exit 143). A big swapfile lets them page to disk instead of being killed. echo "Memory before:" free -h sudo swapoff -a || true diff --git a/.github/workflows/test_models_nightly.yml b/.github/workflows/test_models_nightly.yml index 137386c..39f6c47 100644 --- a/.github/workflows/test_models_nightly.yml +++ b/.github/workflows/test_models_nightly.yml @@ -49,21 +49,11 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Free up disk space - run: | - echo "Disk space before cleanup:" - df -h / - # Reclaim space from preinstalled toolchains we don't use, so there's - # room for a large swapfile and downloaded model weights. - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/boost "$AGENT_TOOLSDIRECTORY" || true - sudo apt-get clean || true - echo "Disk space after cleanup:" - df -h / - name: Increase swap space run: | - # Large models (e.g. phi4 ~14.7B) exceed the 16GB RAM on standard - # runners and get OOM-killed during load/quantization (exit 143). A - # big swapfile lets them page to disk instead of being killed. + # Large models (e.g. gemma-3-4b, Phi-4-mini) push past the 16GB RAM on + # standard runners and get OOM-killed during load/quantization + # (exit 143). A big swapfile lets them page to disk instead of being killed. echo "Memory before:" free -h sudo swapoff -a || true