Skip to content

ci: dual-lane simple-math smoke (no-think + thinking) — from #67 #73

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
antmikinka wants to merge 4 commits into
base: main
Choose a base branch
from
Open

ci: dual-lane simple-math smoke (no-think + thinking) — from #67 #73

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c65fa62
ci: dual-lane simple-math smoke into main (from #67)
antmikinka Jul 28, 2026
1652117
ci: drop HTTP floor probe (nullopt unreachable via chat defaults)
antmikinka Jul 28, 2026
0f4e23b
ci: raise thinking-lane curl timeout; keep max_tokens=512
antmikinka Jul 28, 2026
7d39b63
ci: trim dual-lane smoke comments
antmikinka Jul 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/actions/test-mlx-engine-build/action.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ runs:
"messages": [{"role": "user", "content": "What is 2+2? Reply with just the number."}],
"max_tokens": 32,
"temperature": 0.0,
"enable_thinking": false,
"stream": false
}')

Expand Down
129 changes: 86 additions & 43 deletions .github/workflows/build-mlx-engine.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -378,7 +378,8 @@ jobs:
retention-days: 30

test-simple-math:
name: test-simple-math-${{ matrix.os }}
# Dual lanes: no-think + thinking (matrix.mode).
name: test-simple-math-${{ matrix.os }}-${{ matrix.mode }}
needs: [build-ubuntu-rocm, build-ubuntu-cpu, build-macos]
if: |
always() &&
Expand All @@ -391,49 +392,80 @@ jobs:
fail-fast: false
matrix:
include:
# --- no-think lane ---
- os: ubuntu-rocm
mode: no-think
artifact: mlx-engine-ubuntu-rocm-x64
runs_on: '["self-hosted", "Linux", "X64", "gfx1151"]'
install_rocm: true
hip_device: 0 # gfx1151 APU (HIP device 0); R9700 is device 1
model: mlx-community/Qwen3.5-0.8B-4bit
use_mtp: false
no_think: true
enable_thinking: false
infer_max_tokens: 128
infer_curl_max_time: 180
- os: ubuntu-cpu
mode: no-think
artifact: mlx-engine-ubuntu-cpu-x64
runs_on: '"ubuntu-22.04"'
install_rocm: false
model: mlx-community/Qwen3.5-0.8B-4bit
use_mtp: false
no_think: true
enable_thinking: false
infer_max_tokens: 128
infer_curl_max_time: 180
- os: macos-arm64
mode: no-think
artifact: mlx-engine-macos-arm64
runs_on: '"macos-15"'
install_rocm: false
model: mlx-community/Qwen3.5-0.8B-4bit
use_mtp: false
no_think: true
enable_thinking: false
infer_max_tokens: 128
infer_curl_max_time: 180
# --- thinking lane ---
- os: ubuntu-rocm
mode: thinking
artifact: mlx-engine-ubuntu-rocm-x64
runs_on: '["self-hosted", "Linux", "X64", "gfx1151"]'
install_rocm: true
hip_device: 0
model: mlx-community/Qwen3.5-0.8B-4bit
use_mtp: false
no_think: false
# Temporarily disabled: the GPU MTP smoke test (Qwen3.5-4B-MTP-4bit)
# fails the server/health step on the self-hosted runner. Re-enable
# once MTP inference on ROCm is fixed.
# - os: ubuntu-mtp
# artifact: mlx-engine-ubuntu-rocm-x64
# runs_on: '["self-hosted", "Linux", "X64", "gfx1151"]'
# install_rocm: true
# hip_device: 0 # gfx1151 APU (HIP device 0); R9700 is device 1
# model: mlx-community/Qwen3.5-4B-MTP-4bit
# use_mtp: true
# no_think: true
# timeout_minutes: 30
enable_thinking: true
infer_max_tokens: 4096
infer_curl_max_time: 600
timeout_minutes: 40
- os: ubuntu-cpu
mode: thinking
artifact: mlx-engine-ubuntu-cpu-x64
runs_on: '"ubuntu-22.04"'
install_rocm: false
model: mlx-community/Qwen3.5-0.8B-4bit
use_mtp: false
no_think: false
# Temporarily disabled along with the GPU MTP test (see above).
# - os: ubuntu-cpu-mtp
# artifact: mlx-engine-ubuntu-cpu-x64
# runs_on: '"ubuntu-22.04"'
# install_rocm: false
# model: mlx-community/Qwen3.5-4B-MTP-4bit
# use_mtp: true
# no_think: true
# timeout_minutes: 30
enable_thinking: true
# CPU is slow; curl must cover full 512-token thinking gen.
infer_max_tokens: 512
infer_curl_max_time: 600
timeout_minutes: 35
- os: macos-arm64
mode: thinking
artifact: mlx-engine-macos-arm64
runs_on: '"macos-15"'
install_rocm: false
model: mlx-community/Qwen3.5-0.8B-4bit
use_mtp: false
no_think: false
enable_thinking: true
infer_max_tokens: 512
infer_curl_max_time: 600
timeout_minutes: 35
steps:
- name: ROCm runtime setup (Ubuntu ROCm only)
if: matrix.install_rocm
Expand Down Expand Up @@ -542,11 +574,16 @@ jobs:
MTP_FLAG="--use-mtp --n-draft-tokens 3"
fi

# Disable thinking/reasoning for deterministic, fast answers in smoke tests
NO_THINK_FLAG=""
if [ "${{ matrix.no_think }}" = "true" ]; then
NO_THINK_FLAG="--no-think"
# --no-think when matrix.no_think is true.
NO_THINK_FLAG="--no-think"
if [ "${{ matrix.no_think }}" = "false" ]; then
NO_THINK_FLAG=""
fi
ENABLE_THINKING="${{ matrix.enable_thinking }}"
INFER_MAX_TOKENS="${{ matrix.infer_max_tokens }}"
INFER_CURL_MAX_TIME="${{ matrix.infer_curl_max_time }}"
MODE="${{ matrix.mode }}"
echo "Smoke mode=$MODE no_think_flag=${NO_THINK_FLAG:-<empty>} enable_thinking=$ENABLE_THINKING infer_max_tokens=$INFER_MAX_TOKENS"

# Increase readiness timeout for MTP models
if [ "${{ matrix.use_mtp }}" = "true" ]; then
Expand All @@ -566,7 +603,7 @@ jobs:
echo "All shared libraries resolved OK"
fi

echo "Starting server on port $PORT (MTP=${MTP_FLAG:+ON:-OFF})..."
echo "Starting server on port $PORT (mode=$MODE MTP=${MTP_FLAG:+ON:-OFF})..."
# PR #10 (MLX commit 45f6ee1) fixed affine_dequantize_packed_kernel #pragma unroll
# miscompile on RDNA 3.5. Our local patch was removed — dequant+GEMM path now works
# correctly for uint32 packed weights on ROCm.
Expand Down Expand Up @@ -617,15 +654,16 @@ jobs:
exit 1
fi

# Best-effort model warm-up (discard result)
echo "Sending warm-up request..."
# Warm-up (thinking off).
echo "Sending warm-up request (enable_thinking=false, max_tokens=5)..."
curl -s --max-time 120 -o /dev/null -X POST http://127.0.0.1:$PORT/v1/chat/completions \
-H "Content-Type: application/json" \
-d "{
\"model\": \"${{ matrix.model }}\",
\"messages\": [{\"role\": \"user\", \"content\": \"hi\"}],
\"max_tokens\": 5,
\"temperature\": 0.0,
\"enable_thinking\": false,
\"stream\": false
}" 2>/dev/null || echo "Warm-up request skipped (non-fatal)"

Expand All @@ -636,18 +674,19 @@ jobs:

while [ $ATTEMPT -lt $MAX_RETRIES ]; do
ATTEMPT=$((ATTEMPT + 1))
echo "=== Inference test attempt $ATTEMPT/$MAX_RETRIES ==="
echo "=== Inference test attempt $ATTEMPT/$MAX_RETRIES (mode=$MODE enable_thinking=$ENABLE_THINKING max_tokens=$INFER_MAX_TOKENS) ==="

response=$(curl -s --max-time 180 -X POST http://127.0.0.1:$PORT/v1/chat/completions \
response=$(curl -s --max-time "$INFER_CURL_MAX_TIME" -X POST http://127.0.0.1:$PORT/v1/chat/completions \
-H "Content-Type: application/json" \
-d "{
\"model\": \"${{ matrix.model }}\",
\"messages\": [
{\"role\": \"system\", \"content\": \"You are a math assistant. Answer arithmetic questions with a single number.\"},
{\"role\": \"user\", \"content\": \"What is 2+2? Answer with just the number.\"}
],
\"max_tokens\": 128,
\"max_tokens\": $INFER_MAX_TOKENS,
\"temperature\": 0.0,
\"enable_thinking\": $ENABLE_THINKING,
\"stream\": false
}" 2>&1) || {
echo "Attempt $ATTEMPT: chat completion request failed"
Expand All @@ -660,6 +699,10 @@ jobs:
echo "$response" | python3 -m json.tool 2>/dev/null || echo "$response"
echo "=== END RAW OUTPUT ==="
echo "Attempt $ATTEMPT extracted answer: $answer"
if [ "$ENABLE_THINKING" = "true" ]; then
echo "thinking_mode_signals:"
grep -E 'effective_thinking|thinking_budget_floor' server.log | tail -10 || true
fi

if [ "$answer" = "4" ]; then
PASS=true
Expand All @@ -670,19 +713,19 @@ jobs:
done

if [ "$PASS" = "true" ]; then
echo "INFERENCE TEST PASSED -- model answered correctly on attempt $ATTEMPT"
echo "INFERENCE TEST PASSED -- mode=$MODE answered correctly on attempt $ATTEMPT"
else
echo "=== Server Response ==="
echo "$response" | python3 -m json.tool 2>/dev/null || echo "$response"
echo "=== End Response ==="
echo "INFERENCE TEST FAILED -- expected answer '4' after $MAX_RETRIES attempts"
echo "INFERENCE TEST FAILED -- mode=$MODE expected answer '4' after $MAX_RETRIES attempts"
echo "=== Server Log (last 50 lines) ==="
tail -50 server.log 2>/dev/null || true
stop_server
exit 1
fi

echo "SMOKE TEST PASSED -- server started, health check, and inference OK"
echo "SMOKE TEST PASSED -- mode=$MODE health + inference OK"

# --- Benchmark: extract TPS metrics as structured JSON ---
echo "=== Performance Benchmarks ==="
Expand All @@ -694,7 +737,7 @@ jobs:
from datetime import datetime, timezone

model = '${{ matrix.model }}'
platform = '${{ matrix.os }}'
platform = '${{ matrix.os }}-${{ matrix.mode }}'
ts = ''
try:
with open('server.log', 'r') as f:
Expand Down Expand Up @@ -741,18 +784,18 @@ jobs:
results.append(entry)

if results:
with open('benchmark-results-${{ matrix.os }}.json', 'w') as f:
with open('benchmark-results-${{ matrix.os }}-${{ matrix.mode }}.json', 'w') as f:
json.dump(results, f, indent=2)

# Print formatted datatable
print(f\" {'Platform':<16} {'Prompt TPS':>12} {'Gen TPS':>10} {'Tokens':>8} {'MTP':>6}\")
print(f\" {'-'*14:<16} {'-'*10:>12} {'-'*8:>10} {'-'*6:>8} {'-'*4:>6}\")
print(f\" {'Platform':<24} {'Prompt TPS':>12} {'Gen TPS':>10} {'Tokens':>8} {'MTP':>6}\")
print(f\" {'-'*22:<24} {'-'*10:>12} {'-'*8:>10} {'-'*6:>8} {'-'*4:>6}\")
for r in results:
mtp_status = 'YES' if r.get('mtp_enabled') else 'no'
print(f\" {r['platform']:<16} {r['prompt_tps']:>12.2f} {r['generation_tps']:>10.2f} {r['generation_tokens']:>8} {mtp_status:>6}\")
print(f\" {r['platform']:<24} {r['prompt_tps']:>12.2f} {r['generation_tps']:>10.2f} {r['generation_tokens']:>8} {mtp_status:>6}\")
else:
# Write empty result so artifact still uploads
with open('benchmark-results-${{ matrix.os }}.json', 'w') as f:
with open('benchmark-results-${{ matrix.os }}-${{ matrix.mode }}.json', 'w') as f:
json.dump([], f)
print(' (no TPS data found in server log)')
"
Expand All @@ -764,8 +807,8 @@ jobs:
if: always()
uses: actions/upload-artifact@v6
with:
name: bench-results-${{ matrix.os }}
path: benchmark-results-${{ matrix.os }}.json
name: bench-results-${{ matrix.os }}-${{ matrix.mode }}
path: benchmark-results-${{ matrix.os }}-${{ matrix.mode }}.json
retention-days: 30

collect-benchmarks:
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/test-mlx-engine.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,7 @@ jobs:
"messages": [{"role": "user", "content": "What is 2+2? Reply with just the number."}],
"max_tokens": 32,
"temperature": 0.0,
"enable_thinking": false,
"stream": false
}')

Expand Down Expand Up @@ -371,6 +372,7 @@ jobs:
"messages": [{"role": "user", "content": "What is 2+2? Reply with just the number."}],
"max_tokens": 32,
"temperature": 0.0,
"enable_thinking": false,
"stream": true
}')

Expand Down
Loading