Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,11 @@ jobs:
strategy:
matrix:
include:
- rust: 1.41.1 # MSRV
- rust: 1.75.0 # MSRV
experimental: false
os: ubuntu-latest
target: x86_64-unknown-linux-gnu
features: cgemm
features: threading cgemm
- rust: stable
experimental: false
os: ubuntu-latest
Expand All @@ -40,7 +40,7 @@ jobs:
os: ubuntu-latest
target: x86_64-unknown-linux-gnu
features: threading cgemm
mmtest_feature: fma
mmtest_feature: avx2,fma
experimental: false
- rust: nightly
os: ubuntu-latest
Expand Down Expand Up @@ -101,7 +101,7 @@ jobs:
strategy:
matrix:
include:
- rust: 1.41.1 # MSRV
- rust: 1.75.0 # MSRV
experimental: false
target: thumbv6m-none-eabi
- rust: stable
Expand Down
14 changes: 10 additions & 4 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
@@ -1,18 +1,24 @@

## Guidelines

As a current guiding principle, the entrance functions (i.e sgemm, dgemm etc) are non-generic by design, so that the compile time cost of the library is limited.
As a current guiding principle, the entrance functions (i.e sgemm, dgemm etc)
are non-generic by design, so that the compile time cost of the library is
limited.

Threading is supported using thread-tree for its lower dispatch overhead but it's welcome to replace it by rayon - if the low dispatch overhead can be preserved.
Threading is supported using thread-tree for its lower dispatch overhead but
it's welcome to replace it by rayon - if the low dispatch overhead can be
preserved.

## Test tricks

Use MMTEST_FEATURE=fma and so on to restrict target feature detection to the given feature. Note that this currently only supports a single target feature at a time.
Use MMTEST_FEATURE=avx,fma,avx2 and so on to restrict target feature detection
to the exact listed features: comma-separated list of feature names.

## Benchmarks

To run benchmarks, use `./benches/benchloop.py`

## Wasm

To test and benchmark wasm, add the wasm32-wasip1 target using rustup and install wasmtime-cli.
To test and benchmark wasm, add the wasm32-wasip1 target using rustup and
install wasmtime-cli.
1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ categories = ["science"]
exclude = ["examples/*", "ci/*", ".github/*", "benches/*"]

build = "build.rs"
rust-version = "1.75.0" # MSRV

[lib]
bench = false
Expand Down
7 changes: 0 additions & 7 deletions build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,8 @@ fn main() {
// Avoid `unexpected_cfgs` lint from 1.80+ toolchains
if ac.probe_rustc_version(1, 80) {
println!("cargo:rustc-check-cfg=cfg(has_avx512)");
println!("cargo:rustc-check-cfg=cfg(has_aarch64_simd)");
}

if target_arch == "aarch64" {
// From 1.61 aarch64 intrinsics and #[target_feature]
if ac.probe_rustc_version(1, 61) {
println!("cargo:rustc-cfg=has_aarch64_simd");
}
}
if target_arch == "x86" || target_arch == "x86_64" {
// From 1.89 AVX-512 intrinsics ("avx512f")
if ac.probe_rustc_version(1, 89)
Expand Down
1 change: 0 additions & 1 deletion clippy.toml
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
msrv = "1.41.1"
too-many-arguments-threshold = 20
4 changes: 2 additions & 2 deletions src/aarch64/macros.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ macro_rules! is_aarch64_feature_detected_ {
// For testing purposes, we can make sure only one specific feature
// is enabled by setting MMTEST_FEATURE=featurename (all others
// disabled). This does not force it to be detected, it must also be.
compile_env_matches_or_is_empty!("MMTEST_FEATURE", $name) && std::arch::is_aarch64_feature_detected!($name)
crate::allow_feature($name) && std::arch::is_aarch64_feature_detected!($name)
}
#[cfg(not(feature="std"))]
{
Expand All @@ -15,7 +15,7 @@ macro_rules! is_aarch64_feature_detected_ {
// be. In the `no_std` case, the `is_86_feature_detected` macro is
// not available, so we have to fall back to checking whether the
// feature is enabled at compile-time.
compile_env_matches_or_is_empty!("MMTEST_FEATURE", $name) && cfg!(target_feature=$name)
crate::allow_feature($name) && cfg!(target_feature=$name)
}
}};
}
11 changes: 0 additions & 11 deletions src/archmacros.rs

This file was deleted.

67 changes: 3 additions & 64 deletions src/cgemm_kernel.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,8 @@ use crate::packing::PackSlice;
struct KernelAvx512;
#[cfg(any(target_arch="x86", target_arch="x86_64"))]
struct KernelAvx2;
#[cfg(any(target_arch="x86", target_arch="x86_64"))]
struct KernelFma;

#[cfg(target_arch = "aarch64")]
#[cfg(has_aarch64_simd)]
struct KernelNeon;

struct KernelFallback;
Expand All @@ -48,15 +45,11 @@ pub(crate) fn detect<G>(selector: G) where G: GemmSelect<T> {
return selector.select(KernelAvx512);
}
}
if is_x86_feature_detected_!("fma") {
if is_x86_feature_detected_!("avx2") {
return selector.select(KernelAvx2);
}
return selector.select(KernelFma);
if is_x86_feature_detected_!("fma") && is_x86_feature_detected_!("avx2") {
return selector.select(KernelAvx2);
}
}
#[cfg(target_arch = "aarch64")]
#[cfg(has_aarch64_simd)]
{
if is_aarch64_feature_detected_!("neon") {
return selector.select(KernelNeon);
Expand Down Expand Up @@ -133,42 +126,7 @@ impl GemmKernel for KernelAvx2 {
}
}

#[cfg(any(target_arch="x86", target_arch="x86_64"))]
impl GemmKernel for KernelFma {
type Elem = T;

type MRTy = U4;
type NRTy = U4;

#[inline(always)]
fn align_to() -> usize { 16 }

#[inline(always)]
fn always_masked() -> bool { KernelFallback::always_masked() }

#[inline(always)]
fn nc() -> usize { archparam::C_NC }
#[inline(always)]
fn kc() -> usize { archparam::C_KC }
#[inline(always)]
fn mc() -> usize { archparam::C_MC }

pack_methods!{}

#[inline(always)]
unsafe fn kernel(
k: usize,
alpha: T,
a: *const T,
b: *const T,
beta: T,
c: *mut T, rsc: isize, csc: isize) {
kernel_target_fma(k, alpha, a, b, beta, c, rsc, csc)
}
}

#[cfg(target_arch = "aarch64")]
#[cfg(has_aarch64_simd)]
impl GemmKernel for KernelNeon {
type Elem = T;

Expand Down Expand Up @@ -257,35 +215,18 @@ macro_rules! loop_n { ($j:ident, $e:expr) => { loop4!($j, $e) }; }
#[cfg(any(target_arch="x86", target_arch="x86_64"))]
kernel_fallback_impl_complex! {
// instantiate separately
[inline target_feature(enable="avx2") target_feature(enable="fma")] [fma_yes]
[inline target_feature(enable="fma,avx2")] [fma_yes]
kernel_target_avx2, T, TReal, KernelAvx2::MR, KernelAvx2::NR, 4
}


// Kernel Fma
#[cfg(any(target_arch="x86", target_arch="x86_64"))]
macro_rules! loop_m { ($i:ident, $e:expr) => { loop4!($i, $e) }; }
#[cfg(any(target_arch="x86", target_arch="x86_64"))]
macro_rules! loop_n { ($j:ident, $e:expr) => { loop4!($j, $e) }; }

#[cfg(any(target_arch="x86", target_arch="x86_64"))]
kernel_fallback_impl_complex! {
// instantiate separately
[inline target_feature(enable="fma")] [fma_no]
kernel_target_fma, T, TReal, KernelFma::MR, KernelFma::NR, 2
}

// Kernel neon

#[cfg(target_arch = "aarch64")]
#[cfg(has_aarch64_simd)]
macro_rules! loop_m { ($i:ident, $e:expr) => { loop4!($i, $e) }; }
#[cfg(target_arch = "aarch64")]
#[cfg(has_aarch64_simd)]
macro_rules! loop_n { ($j:ident, $e:expr) => { loop2!($j, $e) }; }

#[cfg(target_arch = "aarch64")]
#[cfg(has_aarch64_simd)]
kernel_fallback_impl_complex! {
[inline target_feature(enable="neon")] [fma_yes]
kernel_target_neon, T, TReal, KernelNeon::MR, KernelNeon::NR, 1
Expand All @@ -312,7 +253,6 @@ mod tests {
}

#[cfg(target_arch = "aarch64")]
#[cfg(has_aarch64_simd)]
mod test_kernel_aarch64 {
use super::test_complex_packed_kernel;
use super::super::*;
Expand Down Expand Up @@ -362,7 +302,6 @@ mod tests {
}

test_arch_kernels_x86! {
"fma", fma, KernelFma,
"avx2", avx2, KernelAvx2
}

Expand Down
57 changes: 57 additions & 0 deletions src/constfind.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
//! Copyright 2026 Ulrik Sverdrup "bluss"

pub(crate) const fn slice_eq(a: &[u8], b: &[u8]) -> bool {
if a.len() != b.len() { return false; }
let mut i = 0;
while i < a.len() {
if a[i] != b[i] { return false; }
i += 1;
}
true
}

const fn find_byte(text: &[u8], byte: u8) -> Option<usize> {
let mut j = 0;
while j < text.len() && text[j] != byte {
j += 1;
}
if j == text.len() { None } else { Some(j) }
}

/// Search for exact word match in string of comma separated words
///
/// Example "avx2,fma", "avx2" => true; "avx2,fma", "avx" => false
pub(crate) const fn comma_separated_contains(text: &str, word: &str) -> bool {
let mut text = text.as_bytes();
loop {
let next_comma = find_byte(text, b',');
let word_end = match next_comma { Some(x) => x, None => text.len() };
let (this_word, _) = text.split_at(word_end);
if slice_eq(this_word, word.as_bytes()) {
return true;
}

// take next segment
if let None = next_comma {
return false;
}
let (_, tail) = text.split_at(word_end + 1);
text = tail;
}
}

#[test]
fn test_find_byte() {
assert_eq!(find_byte(b"abc", b'z'), None);
assert_eq!(find_byte(b"abc", b'b'), Some(1));
}

#[test]
fn test_comma_separated_contains() {
assert!(comma_separated_contains("abc,xyz", "abc"));
assert!(comma_separated_contains("abc,xyz", "xyz"));
assert!(!comma_separated_contains("abc,xyz", "abc,"));
assert!(!comma_separated_contains("avx2,fma", "avx"));
}


Loading
Loading