diff --git a/.github/actions/test-mlx-engine-build/action.yml b/.github/actions/test-mlx-engine-build/action.yml index a5cddd22..943d1084 100644 --- a/.github/actions/test-mlx-engine-build/action.yml +++ b/.github/actions/test-mlx-engine-build/action.yml @@ -82,6 +82,7 @@ runs: "messages": [{"role": "user", "content": "What is 2+2? Reply with just the number."}], "max_tokens": 32, "temperature": 0.0, + "enable_thinking": false, "stream": false }') diff --git a/.github/workflows/build-mlx-engine.yml b/.github/workflows/build-mlx-engine.yml index 94b10d3d..63a97580 100644 --- a/.github/workflows/build-mlx-engine.yml +++ b/.github/workflows/build-mlx-engine.yml @@ -378,7 +378,8 @@ jobs: retention-days: 30 test-simple-math: - name: test-simple-math-${{ matrix.os }} + # Dual lanes: no-think + thinking (matrix.mode). + name: test-simple-math-${{ matrix.os }}-${{ matrix.mode }} needs: [build-ubuntu-rocm, build-ubuntu-cpu, build-macos] if: | always() && @@ -391,49 +392,80 @@ jobs: fail-fast: false matrix: include: + # --- no-think lane --- - os: ubuntu-rocm + mode: no-think artifact: mlx-engine-ubuntu-rocm-x64 runs_on: '["self-hosted", "Linux", "X64", "gfx1151"]' install_rocm: true hip_device: 0 # gfx1151 APU (HIP device 0); R9700 is device 1 model: mlx-community/Qwen3.5-0.8B-4bit use_mtp: false + no_think: true + enable_thinking: false + infer_max_tokens: 128 + infer_curl_max_time: 180 + - os: ubuntu-cpu + mode: no-think + artifact: mlx-engine-ubuntu-cpu-x64 + runs_on: '"ubuntu-22.04"' + install_rocm: false + model: mlx-community/Qwen3.5-0.8B-4bit + use_mtp: false + no_think: true + enable_thinking: false + infer_max_tokens: 128 + infer_curl_max_time: 180 + - os: macos-arm64 + mode: no-think + artifact: mlx-engine-macos-arm64 + runs_on: '"macos-15"' + install_rocm: false + model: mlx-community/Qwen3.5-0.8B-4bit + use_mtp: false + no_think: true + enable_thinking: false + infer_max_tokens: 128 + infer_curl_max_time: 180 + # --- thinking lane --- + - os: ubuntu-rocm + mode: thinking + artifact: mlx-engine-ubuntu-rocm-x64 + runs_on: '["self-hosted", "Linux", "X64", "gfx1151"]' + install_rocm: true + hip_device: 0 + model: mlx-community/Qwen3.5-0.8B-4bit + use_mtp: false no_think: false - # Temporarily disabled: the GPU MTP smoke test (Qwen3.5-4B-MTP-4bit) - # fails the server/health step on the self-hosted runner. Re-enable - # once MTP inference on ROCm is fixed. - # - os: ubuntu-mtp - # artifact: mlx-engine-ubuntu-rocm-x64 - # runs_on: '["self-hosted", "Linux", "X64", "gfx1151"]' - # install_rocm: true - # hip_device: 0 # gfx1151 APU (HIP device 0); R9700 is device 1 - # model: mlx-community/Qwen3.5-4B-MTP-4bit - # use_mtp: true - # no_think: true - # timeout_minutes: 30 + enable_thinking: true + infer_max_tokens: 4096 + infer_curl_max_time: 600 + timeout_minutes: 40 - os: ubuntu-cpu + mode: thinking artifact: mlx-engine-ubuntu-cpu-x64 runs_on: '"ubuntu-22.04"' install_rocm: false model: mlx-community/Qwen3.5-0.8B-4bit use_mtp: false no_think: false - # Temporarily disabled along with the GPU MTP test (see above). - # - os: ubuntu-cpu-mtp - # artifact: mlx-engine-ubuntu-cpu-x64 - # runs_on: '"ubuntu-22.04"' - # install_rocm: false - # model: mlx-community/Qwen3.5-4B-MTP-4bit - # use_mtp: true - # no_think: true - # timeout_minutes: 30 + enable_thinking: true + # CPU is slow; curl must cover full 512-token thinking gen. + infer_max_tokens: 512 + infer_curl_max_time: 600 + timeout_minutes: 35 - os: macos-arm64 + mode: thinking artifact: mlx-engine-macos-arm64 runs_on: '"macos-15"' install_rocm: false model: mlx-community/Qwen3.5-0.8B-4bit use_mtp: false no_think: false + enable_thinking: true + infer_max_tokens: 512 + infer_curl_max_time: 600 + timeout_minutes: 35 steps: - name: ROCm runtime setup (Ubuntu ROCm only) if: matrix.install_rocm @@ -542,11 +574,16 @@ jobs: MTP_FLAG="--use-mtp --n-draft-tokens 3" fi - # Disable thinking/reasoning for deterministic, fast answers in smoke tests - NO_THINK_FLAG="" - if [ "${{ matrix.no_think }}" = "true" ]; then - NO_THINK_FLAG="--no-think" + # --no-think when matrix.no_think is true. + NO_THINK_FLAG="--no-think" + if [ "${{ matrix.no_think }}" = "false" ]; then + NO_THINK_FLAG="" fi + ENABLE_THINKING="${{ matrix.enable_thinking }}" + INFER_MAX_TOKENS="${{ matrix.infer_max_tokens }}" + INFER_CURL_MAX_TIME="${{ matrix.infer_curl_max_time }}" + MODE="${{ matrix.mode }}" + echo "Smoke mode=$MODE no_think_flag=${NO_THINK_FLAG:-} enable_thinking=$ENABLE_THINKING infer_max_tokens=$INFER_MAX_TOKENS" # Increase readiness timeout for MTP models if [ "${{ matrix.use_mtp }}" = "true" ]; then @@ -566,7 +603,7 @@ jobs: echo "All shared libraries resolved OK" fi - echo "Starting server on port $PORT (MTP=${MTP_FLAG:+ON:-OFF})..." + echo "Starting server on port $PORT (mode=$MODE MTP=${MTP_FLAG:+ON:-OFF})..." # PR #10 (MLX commit 45f6ee1) fixed affine_dequantize_packed_kernel #pragma unroll # miscompile on RDNA 3.5. Our local patch was removed — dequant+GEMM path now works # correctly for uint32 packed weights on ROCm. @@ -617,8 +654,8 @@ jobs: exit 1 fi - # Best-effort model warm-up (discard result) - echo "Sending warm-up request..." + # Warm-up (thinking off). + echo "Sending warm-up request (enable_thinking=false, max_tokens=5)..." curl -s --max-time 120 -o /dev/null -X POST http://127.0.0.1:$PORT/v1/chat/completions \ -H "Content-Type: application/json" \ -d "{ @@ -626,6 +663,7 @@ jobs: \"messages\": [{\"role\": \"user\", \"content\": \"hi\"}], \"max_tokens\": 5, \"temperature\": 0.0, + \"enable_thinking\": false, \"stream\": false }" 2>/dev/null || echo "Warm-up request skipped (non-fatal)" @@ -636,9 +674,9 @@ jobs: while [ $ATTEMPT -lt $MAX_RETRIES ]; do ATTEMPT=$((ATTEMPT + 1)) - echo "=== Inference test attempt $ATTEMPT/$MAX_RETRIES ===" + echo "=== Inference test attempt $ATTEMPT/$MAX_RETRIES (mode=$MODE enable_thinking=$ENABLE_THINKING max_tokens=$INFER_MAX_TOKENS) ===" - response=$(curl -s --max-time 180 -X POST http://127.0.0.1:$PORT/v1/chat/completions \ + response=$(curl -s --max-time "$INFER_CURL_MAX_TIME" -X POST http://127.0.0.1:$PORT/v1/chat/completions \ -H "Content-Type: application/json" \ -d "{ \"model\": \"${{ matrix.model }}\", @@ -646,8 +684,9 @@ jobs: {\"role\": \"system\", \"content\": \"You are a math assistant. Answer arithmetic questions with a single number.\"}, {\"role\": \"user\", \"content\": \"What is 2+2? Answer with just the number.\"} ], - \"max_tokens\": 128, + \"max_tokens\": $INFER_MAX_TOKENS, \"temperature\": 0.0, + \"enable_thinking\": $ENABLE_THINKING, \"stream\": false }" 2>&1) || { echo "Attempt $ATTEMPT: chat completion request failed" @@ -660,6 +699,10 @@ jobs: echo "$response" | python3 -m json.tool 2>/dev/null || echo "$response" echo "=== END RAW OUTPUT ===" echo "Attempt $ATTEMPT extracted answer: $answer" + if [ "$ENABLE_THINKING" = "true" ]; then + echo "thinking_mode_signals:" + grep -E 'effective_thinking|thinking_budget_floor' server.log | tail -10 || true + fi if [ "$answer" = "4" ]; then PASS=true @@ -670,19 +713,19 @@ jobs: done if [ "$PASS" = "true" ]; then - echo "INFERENCE TEST PASSED -- model answered correctly on attempt $ATTEMPT" + echo "INFERENCE TEST PASSED -- mode=$MODE answered correctly on attempt $ATTEMPT" else echo "=== Server Response ===" echo "$response" | python3 -m json.tool 2>/dev/null || echo "$response" echo "=== End Response ===" - echo "INFERENCE TEST FAILED -- expected answer '4' after $MAX_RETRIES attempts" + echo "INFERENCE TEST FAILED -- mode=$MODE expected answer '4' after $MAX_RETRIES attempts" echo "=== Server Log (last 50 lines) ===" tail -50 server.log 2>/dev/null || true stop_server exit 1 fi - echo "SMOKE TEST PASSED -- server started, health check, and inference OK" + echo "SMOKE TEST PASSED -- mode=$MODE health + inference OK" # --- Benchmark: extract TPS metrics as structured JSON --- echo "=== Performance Benchmarks ===" @@ -694,7 +737,7 @@ jobs: from datetime import datetime, timezone model = '${{ matrix.model }}' - platform = '${{ matrix.os }}' + platform = '${{ matrix.os }}-${{ matrix.mode }}' ts = '' try: with open('server.log', 'r') as f: @@ -741,18 +784,18 @@ jobs: results.append(entry) if results: - with open('benchmark-results-${{ matrix.os }}.json', 'w') as f: + with open('benchmark-results-${{ matrix.os }}-${{ matrix.mode }}.json', 'w') as f: json.dump(results, f, indent=2) # Print formatted datatable - print(f\" {'Platform':<16} {'Prompt TPS':>12} {'Gen TPS':>10} {'Tokens':>8} {'MTP':>6}\") - print(f\" {'-'*14:<16} {'-'*10:>12} {'-'*8:>10} {'-'*6:>8} {'-'*4:>6}\") + print(f\" {'Platform':<24} {'Prompt TPS':>12} {'Gen TPS':>10} {'Tokens':>8} {'MTP':>6}\") + print(f\" {'-'*22:<24} {'-'*10:>12} {'-'*8:>10} {'-'*6:>8} {'-'*4:>6}\") for r in results: mtp_status = 'YES' if r.get('mtp_enabled') else 'no' - print(f\" {r['platform']:<16} {r['prompt_tps']:>12.2f} {r['generation_tps']:>10.2f} {r['generation_tokens']:>8} {mtp_status:>6}\") + print(f\" {r['platform']:<24} {r['prompt_tps']:>12.2f} {r['generation_tps']:>10.2f} {r['generation_tokens']:>8} {mtp_status:>6}\") else: # Write empty result so artifact still uploads - with open('benchmark-results-${{ matrix.os }}.json', 'w') as f: + with open('benchmark-results-${{ matrix.os }}-${{ matrix.mode }}.json', 'w') as f: json.dump([], f) print(' (no TPS data found in server log)') " @@ -764,8 +807,8 @@ jobs: if: always() uses: actions/upload-artifact@v6 with: - name: bench-results-${{ matrix.os }} - path: benchmark-results-${{ matrix.os }}.json + name: bench-results-${{ matrix.os }}-${{ matrix.mode }} + path: benchmark-results-${{ matrix.os }}-${{ matrix.mode }}.json retention-days: 30 collect-benchmarks: diff --git a/.github/workflows/test-mlx-engine.yml b/.github/workflows/test-mlx-engine.yml index 21bdd9c9..416f7f83 100644 --- a/.github/workflows/test-mlx-engine.yml +++ b/.github/workflows/test-mlx-engine.yml @@ -303,6 +303,7 @@ jobs: "messages": [{"role": "user", "content": "What is 2+2? Reply with just the number."}], "max_tokens": 32, "temperature": 0.0, + "enable_thinking": false, "stream": false }') @@ -371,6 +372,7 @@ jobs: "messages": [{"role": "user", "content": "What is 2+2? Reply with just the number."}], "max_tokens": 32, "temperature": 0.0, + "enable_thinking": false, "stream": true }')