MTP: wire delta kernel, probabilistic acceptance, cache prefill + MLX upstream sync

.github/workflows/build-mlx-engine.yml

@@ -72,7 +72,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v6
+      uses: actions/checkout@v4
       with:
         submodules: recursive
 
@@ -244,7 +244,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v6
+      uses: actions/checkout@v4
       with:
         submodules: recursive
 
@@ -311,7 +311,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v6
+      uses: actions/checkout@v4
       with:
         submodules: recursive
 
@@ -877,7 +877,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v6
+      uses: actions/checkout@v4
 
     - name: Download all build artifacts
       uses: actions/download-artifact@v6

.github/workflows/pr-agent.yml

@@ -0,0 +1,34 @@
+name: PR-Agent
+on:
+  pull_request_target:
+    types: [opened, ready_for_review, reopened]
+  issue_comment:
+    types: [created, edited]
+
+permissions:
+  pull-requests: write
+  issues: write
+  contents: read
+
+jobs:
+  pr-agent:
+    runs-on: ubuntu-latest
+    if: |
+      (github.event_name == 'pull_request_target' && github.event.action != 'closed') ||
+      (github.event_name == 'issue_comment' && github.event.issue.pull_request &&
+       contains(github.event.comment.body, '/describe') ||
+       contains(github.event.comment.body, '/review') ||
+       contains(github.event.comment.body, '/improve') ||
+       contains(github.event.comment.body, '/ask'))
+    steps:
+      - name: PR-Agent
+        uses: codium-ai/pr-agent@v0.38.0
+        env:
+          OPENAI_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
+          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          github_action_config.auto_describe: "true"
+          github_action_config.auto_review: "true"
+          github_action_config.auto_improve: "false"
+          github_action_config.enable_output: "true"
+          pr_agent.extra_config: ".pr_agent.toml"

.github/workflows/sync-upstream.yml

@@ -0,0 +1,18 @@
+name: Sync fork with upstream
+
+on:
+  schedule:
+    - cron: '0 6 * * *'
+  workflow_dispatch: {}
+
+permissions:
+  contents: write
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Sync fork with upstream
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: gh repo sync "${{ github.repository }}" --source "lemonade-sdk/lemon-mlx-engine"

.github/workflows/test-mlx-engine.yml

@@ -33,7 +33,7 @@ jobs:
       latest_release_tag: ${{ steps.get-release.outputs.latest_release_tag }}
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v6
+      uses: actions/checkout@v4
 
     - name: Get release tag
       id: get-release
@@ -98,7 +98,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v6
+      uses: actions/checkout@v4
 
     - name: Install jq
       run: |

.gitignore

@@ -42,3 +42,4 @@ Thumbs.db
 .Spotlight-V100
 .Trashes
 build-ci/
+start-mlx-server.sh

.pr_agent.toml

@@ -0,0 +1,33 @@
+[config]
+model = "ollama_chat/gpt-oss:20b"
+fallback_models = ["ollama_chat/qwen3.5:9b"]
+model_turbo = "ollama_chat/gpt-oss:20b"
+verbosity_level = 2
+publish_output = false
+publish_output_progress = false
+git_provider = "github"
+skip_keys = []
+use_repo_settings_file = false
+use_wiki_settings_file = false
+use_global_settings_file = false
+
+[pr_reviewer]
+require_score_review = true
+require_tests_review = true
+num_ais = 1
+inline_code_comments = true
+automatic_review = false
+persistent_comment = false
+extra_instructions = "Focus on INT8 quantization correctness, NPU context lifecycle bugs, BFP16 precision issues, and C++ memory safety. Reviewer: think like an NPU hardware engineer reviewing inference engine code."
+
+[pr_description]
+publish_labels = true
+extra_instructions = "Use conventional commits. Include performance impact (ms/tok delta). Tag NPU-specific changes with [npu]."
+enable_help_text = false
+
+[pr_code_suggestions]
+num_code_suggestions = 4
+extra_instructions = "Prefer suggestions that reduce context-swapping overhead, eliminate heap allocations in decode loop, or improve INT8 quantization accuracy."
+
+[pr_code_suggestions_reflect]
+extra_instructions = "Reflect on whether suggested code would actually improve tok/s or token quality on real NPU hardware. Discard suggestions that are purely stylistic."

CMakeLists.txt

@@ -116,12 +116,12 @@ add_library(mlx-lm-common
     src/common/chat_template.cpp
     src/common/gated_delta.cpp
     src/common/graph_decode.cpp
+    src/common/mtp_delta_kernel.cpp
     src/llm/models/mtp_head.cpp
     src/llm/models/mtp_moe.cpp
 )
-# NOTE: src/common/mtp_delta_kernel.cpp intentionally excluded from build.
-# Contains dead code (mtp_delta_fused / mtp_draft_forward) not wired into
-# generate.cpp. Left on disk for future ROCm optimization work.
+# MTP delta kernel -- fused GatedDeltaNet compute for MTP draft generation.
+# Wired into generate.cpp via the mtp_head and context pipelines.
 target_link_libraries(mlx-lm-common PUBLIC
     mlx-lm-core
     nlohmann_json::nlohmann_json

README.md

@@ -2,15 +2,20 @@
 
 C++ inference engine for large language models, built on [MLX](https://github.com/ml-explore/mlx).
 
-Run LLMs locally on **Apple M-series**, **AMD GPUs** (Linux/Windows), and CPU -- no Python required.
+Run LLMs locally on **Apple M-series**, **AMD GPUs** (Linux/Windows), **AMD XDNA2 NPU** (Strix Halo), and CPU -- no Python required.
 
 ## Features
 
 - **50+ LLM architectures** -- Llama, Qwen, Gemma, Phi, DeepSeek, Mistral, Granite, GLM, Falcon, and more
 - **12 VLM architectures** -- Qwen-VL, PaliGemma, Pixtral, Gemma3, SmolVLM, and more
 - **Embedders** -- BERT, Nomic-BERT, Qwen3-Embed
-- **Quantized inference** -- 4-bit/8-bit via `quantized_matmul`
-- **HuggingFace integration** -- auto-downloads models, tokenizers, and chat templates
+- **1-bit / ternary model support** -- BitNet b1.58, Falcon-E (BitLinear), Bonsai, full 1.58-bit and 1-bit variants
+- **GGUF quant format support** -- Q4_0 through Q6_K, K-quants, auto-quantize on load
+- **NPU backend** -- AMD XDNA2 NPU (Strix Halo) via XRT, custom BFP16 xclbins, zero-copy dispatch
+- **ROCm GPU optimization** -- HIP graph decode, fused MoE/GDN kernels, O(1) memory reuse
+- **Multi-token prediction (MTP)** -- Speculative decoding with MTP head support (Qwen3.5-Next)
+- **Quantized inference** -- 4-bit/8-bit via `quantized_matmul`, KV cache quantization (4/8-bit)
+- **Universal HuggingFace loading** -- auto-quantize, GGUF, PyTorch → safetensors converter
 - **fastokens** -- high-performance BPE tokenizer ([crusoecloud/fastokens](https://github.com/crusoecloud/fastokens))
 - **OpenAI-compatible API server** -- drop-in replacement for local inference
 - **Streaming generation** -- async token pipeline with KV caching
@@ -98,6 +103,24 @@ eGPU link. When mixing a discrete RDNA 4 GPU with an integrated APU, make sure
 `HSA_OVERRIDE_GFX_VERSION` is **unset** so kernels compile for each GPU's real
 architecture (`chat` clears it automatically).
 
+### NPU Backend (AMD XDNA2)
+
+The engine also supports the **AMD XDNA2 NPU** on Strix Halo (Ryzen AI MAX 300)
+via the XRT runtime. The NPU backend dispatches quantized GEMM to format-specific
+BFP16 xclbins on the NPU, with zero-copy memory access via shared UMA.
+
+Build with NPU support:
+
+```bash
+mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Release -DMLX_LM_BUILD_NPU=ON
+make -j
+```
+
+Requires the XRT development headers and the amdxdna kernel driver. The NPU
+backend auto-selects between Q4NX, FP16, and BitNet xclbins based on weight
+format, falling back to CPU/GPU if the NPU is unavailable.
+
 ## API Server
 
 ```