feat(cpu): add Qwen3.5 0.8B mobile CPU support

examples/CMakeLists.txt

@@ -6,6 +6,7 @@ add_subdirectory(llama)
 add_subdirectory(minicpm_o)
 add_subdirectory(minicpm4)
 add_subdirectory(qwen3)
+add_subdirectory(qwen3_5)
 add_subdirectory(qwen3_service)
 add_subdirectory(qwen3_moe)
 add_subdirectory(deepseek_ocr)

examples/qwen3_5/CMakeLists.txt

@@ -0,0 +1,3 @@
+add_executable(mllm-qwen3-5-runner main.cpp)
+target_link_libraries(mllm-qwen3-5-runner PRIVATE MllmRT MllmCPUBackend)
+target_include_directories(mllm-qwen3-5-runner PRIVATE ${MLLM_INCLUDE_DIR})

examples/qwen3_5/README.md

@@ -0,0 +1,63 @@
+# Qwen3.5 0.8B on ARM CPU
+
+This example runs the text tower of `Qwen/Qwen3.5-0.8B`. The checkpoint also
+contains a vision tower and an MTP layer; they are intentionally excluded from
+the CPU model file.
+
+The model uses six full-attention layers and eighteen Gated Delta Net (GDN)
+layers. Both the GDN recurrence and its depthwise-convolution history are
+stateful across prefill and decode. `Qwen3_5ForCausalLM::resetState()` clears
+those states together with the full-attention KV cache.
+
+## Convert the checkpoint
+
+First verify that the checkpoint architecture, tensor shapes, and quantization
+coverage match this 0.8B CPU implementation:
+
+```bash
+python examples/qwen3_5/validate_checkpoint.py \
+  /path/to/Qwen3.5-0.8B
+```
+
+Run the converter from the repository root:
+
+```bash
+python -m pymllm.mobile.utils.mllm_convertor \
+  --input_path /path/to/Qwen3.5-0.8B \
+  --output_path /path/to/qwen3.5-0.8b-w4a32-kai.mllm \
+  --model_name Qwen3.5-0.8B \
+  --cfg_path examples/qwen3_5/quant_cfg_0.8B_w4a32_kai.json \
+  --pipeline w4a32_kai_pipeline \
+  --include_prefix model.language_model. \
+  --format v2 \
+  --verbose
+```
+
+The tied embedding matrix is retained for token lookup and separately packed as
+`lm_head_out.weight` for KAI. Every `nn::Linear`, including the small GDN
+`in_proj_a` and `in_proj_b` gates, is packed for the configured KAI runtime;
+convolution weights, recurrent parameters, embeddings, and norms stay in
+float32.
+
+Audit the resulting V2 descriptors without loading the tensor data:
+
+```bash
+python examples/qwen3_5/validate_converted_model.py \
+  /path/to/qwen3.5-0.8b-w4a32-kai.mllm \
+  /path/to/Qwen3.5-0.8B
+```
+
+## Run
+
+```bash
+mllm-qwen3-5-runner \
+  --model_path /path/to/qwen3.5-0.8b-w4a32-kai.mllm \
+  --model_version v2 \
+  --tokenizer_path /path/to/Qwen3.5-0.8B/tokenizer.json \
+  --config_path examples/qwen3_5/config_0.8B_w4a32_kai.json \
+  --prompt "Give a one-sentence introduction." \
+  --max_new_tokens 32
+```
+
+Omit `--prompt` for the interactive loop. The CLI treats each prompt as an
+independent conversation and resets all model state before inference.

examples/qwen3_5/config_0.8B_w4a32_kai.json

@@ -0,0 +1,67 @@
+{
+  "architectures": [
+    "Qwen3_5ForConditionalGeneration"
+  ],
+  "model_type": "qwen3_5",
+  "text_config": {
+    "attention_bias": false,
+    "attn_output_gate": true,
+    "eos_token_id": 248044,
+    "full_attention_interval": 4,
+    "head_dim": 256,
+    "hidden_size": 1024,
+    "intermediate_size": 3584,
+    "layer_types": [
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention"
+    ],
+    "linear_conv_kernel_dim": 4,
+    "linear_key_head_dim": 128,
+    "linear_num_key_heads": 16,
+    "linear_num_value_heads": 16,
+    "linear_value_head_dim": 128,
+    "max_position_embeddings": 262144,
+    "num_attention_heads": 8,
+    "num_hidden_layers": 24,
+    "num_key_value_heads": 2,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "mrope_interleaved": true,
+      "mrope_section": [
+        11,
+        11,
+        10
+      ],
+      "partial_rotary_factor": 0.25,
+      "rope_theta": 10000000,
+      "rope_type": "default"
+    },
+    "tie_word_embeddings": true,
+    "vocab_size": 248320
+  },
+  "tie_word_embeddings": true,
+  "max_cache_length": 2048,
+  "linear_impl_type": "KaiLinear_f32_qai8dxp_qsi4c32p_mxk_nxk_qai8dxp1x8_qsi4c32p8x8_1x8x32"
+}

examples/qwen3_5/main.cpp

@@ -0,0 +1,121 @@
+#include <fmt/core.h>
+#include <cstdio>
+#include <iostream>
+#include <stdexcept>
+#include <string>
+
+#include <mllm/mllm.hpp>
+#include <mllm/models/qwen3_5/modeling_qwen3_5.hpp>
+#include <mllm/models/qwen3_5/tokenization_qwen3_5.hpp>
+#include <mllm/preprocessor/tokenizers/Unicode.hpp>
+#include <mllm/utils/AnyValue.hpp>
+
+using mllm::Argparse;
+
+MLLM_MAIN({
+  auto& help = Argparse::add<bool>("-h|--help").help("Show help message");
+  auto& model_path = Argparse::add<std::string>("-m|--model_path").help("Model path").required(true);
+  auto& model_version = Argparse::add<std::string>("-mv|--model_version").help("Model version").required(true);
+  auto& tokenizer_path = Argparse::add<std::string>("-t|--tokenizer_path").help("Tokenizer JSON path").required(true);
+  auto& config_path = Argparse::add<std::string>("-c|--config_path").help("Config path").required(true);
+  auto& prompt = Argparse::add<std::string>("-p|--prompt").help("Run one prompt non-interactively").required(false);
+  auto& max_new_tokens = Argparse::add<int>("-g|--max_new_tokens").help("Maximum generated tokens per prompt").required(false);
+  auto& print_token_ids = Argparse::add<bool>("--print_token_ids").help("Print generated token IDs to stderr").required(false);
+
+  // Argparse validates required options during parse(), so short-circuit help
+  // before parsing to make `mllm-qwen3-5-runner --help` usable on its own.
+  for (int i = 1; i < argc; ++i) {
+    if (std::string(argv[i]) == "-h" || std::string(argv[i]) == "--help") {
+      Argparse::printHelp();
+      return 0;
+    }
+  }
+
+  Argparse::parse(argc, argv);
+
+  (void)help;
+
+#ifdef MLLM_PERFETTO_ENABLE
+  mllm::perf::start();
+#endif
+
+  int exit_code = 0;
+  {
+    mllm::ModelFileVersion file_version;
+    if (model_version.get() == "v1") {
+      file_version = mllm::ModelFileVersion::kV1;
+    } else if (model_version.get() == "v2") {
+      file_version = mllm::ModelFileVersion::kV2;
+    } else {
+      throw std::invalid_argument("model_version must be either v1 or v2");
+    }
+
+    auto cfg = mllm::models::qwen3_5::Qwen3_5Config(config_path.get());
+    auto tokenizer = mllm::models::qwen3_5::Qwen3_5Tokenizer(tokenizer_path.get());
+    auto model = mllm::models::qwen3_5::Qwen3_5ForCausalLM(cfg);
+    int generation_limit = max_new_tokens.isSet() ? max_new_tokens.get() : 64;
+    if (generation_limit <= 0 || generation_limit > cfg.max_cache_length) {
+      throw std::invalid_argument("max_new_tokens must be between 1 and max_cache_length");
+    }
+    if (prompt.isSet() && prompt.get().empty()) { throw std::invalid_argument("prompt must not be empty"); }
+
+    fmt::print("Qwen3.5 0.8B: {} layers ({} full attention + {} GDN)\n", cfg.num_hidden_layers, cfg.numFullAttentionLayers(),
+               cfg.numGDNLayers());
+
+    auto param = mllm::load(model_path.get(), file_version);
+    model.load(param);
+
+    fmt::print("\n{:*^60}\n", prompt.isSet() ? " Qwen3.5 One-shot CLI " : " Qwen3.5 Interactive CLI ");
+    if (!prompt.isSet()) { fmt::print("Enter 'exit' or 'quit' to end the session\n\n"); }
+
+    while (true) {
+      std::string prompt_text = prompt.isSet() ? prompt.get() : "";
+      if (!prompt.isSet()) {
+        fmt::print("Prompt text (or 'exit/quit'): ");
+        if (!std::getline(std::cin, prompt_text) || prompt_text == "exit" || prompt_text == "quit") { break; }
+      }
+      if (prompt_text.empty()) { continue; }
+
+      try {
+        // Each prompt is an independent conversation. Both the full-attention
+        // KV cache and every GDN recurrent/conv state must start empty.
+        model.resetState();
+        fmt::print("Processing...\n");
+        auto inputs = tokenizer.convertMessage({.prompt = prompt_text});
+        const auto prompt_length = inputs.at("sequence").shape()[1];
+        if (prompt_length + generation_limit - 1 > cfg.max_cache_length) {
+          throw std::invalid_argument(fmt::format("prompt token count ({}) plus max_new_tokens ({}) exceeds "
+                                                  "max_cache_length ({})",
+                                                  prompt_length, generation_limit, cfg.max_cache_length));
+        }
+
+        fmt::print("\nResponse: ");
+        mllm::models::qwen3_5::Qwen3_5StreamingUtf8Decoder utf8_decoder;
+
+        for (auto& step : model.chat(inputs, {{"max_length", mllm::AnyValue(generation_limit)}})) {
+          if (print_token_ids.isSet() && print_token_ids.get()) { fmt::print(stderr, "TOKEN_ID:{}\n", step.cur_token_id); }
+          fmt::print("{}", utf8_decoder.append(tokenizer.detokenizeBytes(step.cur_token_id)));
+          std::fflush(stdout);
+        }
+        fmt::print("{}", utf8_decoder.finish());
+
+        fmt::print("\n{}\n", std::string(60, '-'));
+      } catch (const std::exception& e) {
+        fmt::print("\nError: {}\n{}\n", e.what(), std::string(60, '-'));
+        if (prompt.isSet()) { exit_code = 1; }
+      }
+      if (prompt.isSet()) { break; }
+    }
+
+    model.perfSummary();
+  }
+
+#ifdef MLLM_PERFETTO_ENABLE
+  mllm::perf::stop();
+  mllm::perf::saveReport("qwen3_5.perf");
+#endif
+
+  mllm::print("\n");
+  mllm::memoryReport();
+  return exit_code;
+})