diff --git a/.github/workflows/test_models.yml b/.github/workflows/test_models.yml index 20f5474..3611f25 100644 --- a/.github/workflows/test_models.yml +++ b/.github/workflows/test_models.yml @@ -42,6 +42,9 @@ jobs: # Custom job name, now shortened and cleaner name: ${{ matrix.test-modeling }} (et=${{ matrix.executorch-version }}, py=${{ matrix.python-version }}, ${{ matrix.os }}) runs-on: ${{ matrix.os }} + # Fail cleanly on a hang instead of dragging the run for hours; large-model + # export can be slow (especially once paging to swap), so keep this generous. + timeout-minutes: 180 env: MODEL_NAME: ${{ matrix.test-modeling }} steps: @@ -50,6 +53,21 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + - name: Increase swap space + run: | + # Large models (e.g. gemma-3-4b, Phi-4-mini) push past the 16GB RAM on + # standard runners and get OOM-killed during load/quantization + # (exit 143). A big swapfile lets them page to disk instead of being killed. + echo "Memory before:" + free -h + sudo swapoff -a || true + sudo rm -f /mnt/swapfile || true + sudo fallocate -l 24G /mnt/swapfile || sudo dd if=/dev/zero of=/mnt/swapfile bs=1M count=24576 + sudo chmod 600 /mnt/swapfile + sudo mkswap /mnt/swapfile + sudo swapon /mnt/swapfile + echo "Memory after:" + free -h - name: Install dependencies for ExecuTorch run: | # Clean up cache to save space diff --git a/.github/workflows/test_models_nightly.yml b/.github/workflows/test_models_nightly.yml index aabc021..39f6c47 100644 --- a/.github/workflows/test_models_nightly.yml +++ b/.github/workflows/test_models_nightly.yml @@ -38,6 +38,9 @@ jobs: name: ${{ matrix.test-modeling }} (et=nightly, py=${{ matrix.python-version }}, ${{ matrix.os }}) runs-on: ${{ matrix.os }} + # Fail cleanly on a hang instead of dragging the run for hours; large-model + # export can be slow (especially once paging to swap), so keep this generous. + timeout-minutes: 180 env: MODEL_NAME: ${{ matrix.test-modeling }} steps: @@ -46,6 +49,21 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + - name: Increase swap space + run: | + # Large models (e.g. gemma-3-4b, Phi-4-mini) push past the 16GB RAM on + # standard runners and get OOM-killed during load/quantization + # (exit 143). A big swapfile lets them page to disk instead of being killed. + echo "Memory before:" + free -h + sudo swapoff -a || true + sudo rm -f /mnt/swapfile || true + sudo fallocate -l 24G /mnt/swapfile || sudo dd if=/dev/zero of=/mnt/swapfile bs=1M count=24576 + sudo chmod 600 /mnt/swapfile + sudo mkswap /mnt/swapfile + sudo swapon /mnt/swapfile + echo "Memory after:" + free -h - name: Install dependencies for ExecuTorch nightly run: | pip cache purge || true