diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 26a8773..b459d62 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,11 +19,11 @@ jobs: strategy: matrix: include: - - rust: 1.41.1 # MSRV + - rust: 1.75.0 # MSRV experimental: false os: ubuntu-latest target: x86_64-unknown-linux-gnu - features: cgemm + features: threading cgemm - rust: stable experimental: false os: ubuntu-latest @@ -40,7 +40,7 @@ jobs: os: ubuntu-latest target: x86_64-unknown-linux-gnu features: threading cgemm - mmtest_feature: fma + mmtest_feature: avx2,fma experimental: false - rust: nightly os: ubuntu-latest @@ -101,7 +101,7 @@ jobs: strategy: matrix: include: - - rust: 1.41.1 # MSRV + - rust: 1.75.0 # MSRV experimental: false target: thumbv6m-none-eabi - rust: stable diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9ae7931..98e0ed2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,13 +1,18 @@ ## Guidelines -As a current guiding principle, the entrance functions (i.e sgemm, dgemm etc) are non-generic by design, so that the compile time cost of the library is limited. +As a current guiding principle, the entrance functions (i.e sgemm, dgemm etc) +are non-generic by design, so that the compile time cost of the library is +limited. -Threading is supported using thread-tree for its lower dispatch overhead but it's welcome to replace it by rayon - if the low dispatch overhead can be preserved. +Threading is supported using thread-tree for its lower dispatch overhead but +it's welcome to replace it by rayon - if the low dispatch overhead can be +preserved. ## Test tricks -Use MMTEST_FEATURE=fma and so on to restrict target feature detection to the given feature. Note that this currently only supports a single target feature at a time. +Use MMTEST_FEATURE=avx,fma,avx2 and so on to restrict target feature detection +to the exact listed features: comma-separated list of feature names. ## Benchmarks @@ -15,4 +20,5 @@ To run benchmarks, use `./benches/benchloop.py` ## Wasm -To test and benchmark wasm, add the wasm32-wasip1 target using rustup and install wasmtime-cli. +To test and benchmark wasm, add the wasm32-wasip1 target using rustup and +install wasmtime-cli. diff --git a/Cargo.toml b/Cargo.toml index 0a60c01..f522e6b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ categories = ["science"] exclude = ["examples/*", "ci/*", ".github/*", "benches/*"] build = "build.rs" +rust-version = "1.75.0" # MSRV [lib] bench = false diff --git a/build.rs b/build.rs index ac52cbf..e822bd9 100644 --- a/build.rs +++ b/build.rs @@ -15,15 +15,8 @@ fn main() { // Avoid `unexpected_cfgs` lint from 1.80+ toolchains if ac.probe_rustc_version(1, 80) { println!("cargo:rustc-check-cfg=cfg(has_avx512)"); - println!("cargo:rustc-check-cfg=cfg(has_aarch64_simd)"); } - if target_arch == "aarch64" { - // From 1.61 aarch64 intrinsics and #[target_feature] - if ac.probe_rustc_version(1, 61) { - println!("cargo:rustc-cfg=has_aarch64_simd"); - } - } if target_arch == "x86" || target_arch == "x86_64" { // From 1.89 AVX-512 intrinsics ("avx512f") if ac.probe_rustc_version(1, 89) diff --git a/clippy.toml b/clippy.toml index fdb5ee2..0d4e02f 100644 --- a/clippy.toml +++ b/clippy.toml @@ -1,2 +1 @@ -msrv = "1.41.1" too-many-arguments-threshold = 20 diff --git a/src/aarch64/macros.rs b/src/aarch64/macros.rs index 8511cc4..3d83343 100644 --- a/src/aarch64/macros.rs +++ b/src/aarch64/macros.rs @@ -5,7 +5,7 @@ macro_rules! is_aarch64_feature_detected_ { // For testing purposes, we can make sure only one specific feature // is enabled by setting MMTEST_FEATURE=featurename (all others // disabled). This does not force it to be detected, it must also be. - compile_env_matches_or_is_empty!("MMTEST_FEATURE", $name) && std::arch::is_aarch64_feature_detected!($name) + crate::allow_feature($name) && std::arch::is_aarch64_feature_detected!($name) } #[cfg(not(feature="std"))] { @@ -15,7 +15,7 @@ macro_rules! is_aarch64_feature_detected_ { // be. In the `no_std` case, the `is_86_feature_detected` macro is // not available, so we have to fall back to checking whether the // feature is enabled at compile-time. - compile_env_matches_or_is_empty!("MMTEST_FEATURE", $name) && cfg!(target_feature=$name) + crate::allow_feature($name) && cfg!(target_feature=$name) } }}; } diff --git a/src/archmacros.rs b/src/archmacros.rs deleted file mode 100644 index dbdde8f..0000000 --- a/src/archmacros.rs +++ /dev/null @@ -1,11 +0,0 @@ - -#[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch="aarch64"))] -macro_rules! compile_env_matches_or_is_empty { - ($envvar:tt, $feature_name:tt) => { - (match option_env!($envvar) { - None => true, - Some(v) => v == $feature_name - }) - } -} - diff --git a/src/cgemm_kernel.rs b/src/cgemm_kernel.rs index 1c0bf64..a8d3362 100644 --- a/src/cgemm_kernel.rs +++ b/src/cgemm_kernel.rs @@ -20,11 +20,8 @@ use crate::packing::PackSlice; struct KernelAvx512; #[cfg(any(target_arch="x86", target_arch="x86_64"))] struct KernelAvx2; -#[cfg(any(target_arch="x86", target_arch="x86_64"))] -struct KernelFma; #[cfg(target_arch = "aarch64")] -#[cfg(has_aarch64_simd)] struct KernelNeon; struct KernelFallback; @@ -48,15 +45,11 @@ pub(crate) fn detect(selector: G) where G: GemmSelect { return selector.select(KernelAvx512); } } - if is_x86_feature_detected_!("fma") { - if is_x86_feature_detected_!("avx2") { - return selector.select(KernelAvx2); - } - return selector.select(KernelFma); + if is_x86_feature_detected_!("fma") && is_x86_feature_detected_!("avx2") { + return selector.select(KernelAvx2); } } #[cfg(target_arch = "aarch64")] - #[cfg(has_aarch64_simd)] { if is_aarch64_feature_detected_!("neon") { return selector.select(KernelNeon); @@ -133,42 +126,7 @@ impl GemmKernel for KernelAvx2 { } } -#[cfg(any(target_arch="x86", target_arch="x86_64"))] -impl GemmKernel for KernelFma { - type Elem = T; - - type MRTy = U4; - type NRTy = U4; - - #[inline(always)] - fn align_to() -> usize { 16 } - - #[inline(always)] - fn always_masked() -> bool { KernelFallback::always_masked() } - - #[inline(always)] - fn nc() -> usize { archparam::C_NC } - #[inline(always)] - fn kc() -> usize { archparam::C_KC } - #[inline(always)] - fn mc() -> usize { archparam::C_MC } - - pack_methods!{} - - #[inline(always)] - unsafe fn kernel( - k: usize, - alpha: T, - a: *const T, - b: *const T, - beta: T, - c: *mut T, rsc: isize, csc: isize) { - kernel_target_fma(k, alpha, a, b, beta, c, rsc, csc) - } -} - #[cfg(target_arch = "aarch64")] -#[cfg(has_aarch64_simd)] impl GemmKernel for KernelNeon { type Elem = T; @@ -257,35 +215,18 @@ macro_rules! loop_n { ($j:ident, $e:expr) => { loop4!($j, $e) }; } #[cfg(any(target_arch="x86", target_arch="x86_64"))] kernel_fallback_impl_complex! { // instantiate separately - [inline target_feature(enable="avx2") target_feature(enable="fma")] [fma_yes] + [inline target_feature(enable="fma,avx2")] [fma_yes] kernel_target_avx2, T, TReal, KernelAvx2::MR, KernelAvx2::NR, 4 } - -// Kernel Fma -#[cfg(any(target_arch="x86", target_arch="x86_64"))] -macro_rules! loop_m { ($i:ident, $e:expr) => { loop4!($i, $e) }; } -#[cfg(any(target_arch="x86", target_arch="x86_64"))] -macro_rules! loop_n { ($j:ident, $e:expr) => { loop4!($j, $e) }; } - -#[cfg(any(target_arch="x86", target_arch="x86_64"))] -kernel_fallback_impl_complex! { - // instantiate separately - [inline target_feature(enable="fma")] [fma_no] - kernel_target_fma, T, TReal, KernelFma::MR, KernelFma::NR, 2 -} - // Kernel neon #[cfg(target_arch = "aarch64")] -#[cfg(has_aarch64_simd)] macro_rules! loop_m { ($i:ident, $e:expr) => { loop4!($i, $e) }; } #[cfg(target_arch = "aarch64")] -#[cfg(has_aarch64_simd)] macro_rules! loop_n { ($j:ident, $e:expr) => { loop2!($j, $e) }; } #[cfg(target_arch = "aarch64")] -#[cfg(has_aarch64_simd)] kernel_fallback_impl_complex! { [inline target_feature(enable="neon")] [fma_yes] kernel_target_neon, T, TReal, KernelNeon::MR, KernelNeon::NR, 1 @@ -312,7 +253,6 @@ mod tests { } #[cfg(target_arch = "aarch64")] - #[cfg(has_aarch64_simd)] mod test_kernel_aarch64 { use super::test_complex_packed_kernel; use super::super::*; @@ -362,7 +302,6 @@ mod tests { } test_arch_kernels_x86! { - "fma", fma, KernelFma, "avx2", avx2, KernelAvx2 } diff --git a/src/constfind.rs b/src/constfind.rs new file mode 100644 index 0000000..9cff5e7 --- /dev/null +++ b/src/constfind.rs @@ -0,0 +1,57 @@ +//! Copyright 2026 Ulrik Sverdrup "bluss" + +pub(crate) const fn slice_eq(a: &[u8], b: &[u8]) -> bool { + if a.len() != b.len() { return false; } + let mut i = 0; + while i < a.len() { + if a[i] != b[i] { return false; } + i += 1; + } + true +} + +const fn find_byte(text: &[u8], byte: u8) -> Option { + let mut j = 0; + while j < text.len() && text[j] != byte { + j += 1; + } + if j == text.len() { None } else { Some(j) } +} + +/// Search for exact word match in string of comma separated words +/// +/// Example "avx2,fma", "avx2" => true; "avx2,fma", "avx" => false +pub(crate) const fn comma_separated_contains(text: &str, word: &str) -> bool { + let mut text = text.as_bytes(); + loop { + let next_comma = find_byte(text, b','); + let word_end = match next_comma { Some(x) => x, None => text.len() }; + let (this_word, _) = text.split_at(word_end); + if slice_eq(this_word, word.as_bytes()) { + return true; + } + + // take next segment + if let None = next_comma { + return false; + } + let (_, tail) = text.split_at(word_end + 1); + text = tail; + } +} + +#[test] +fn test_find_byte() { + assert_eq!(find_byte(b"abc", b'z'), None); + assert_eq!(find_byte(b"abc", b'b'), Some(1)); +} + +#[test] +fn test_comma_separated_contains() { + assert!(comma_separated_contains("abc,xyz", "abc")); + assert!(comma_separated_contains("abc,xyz", "xyz")); + assert!(!comma_separated_contains("abc,xyz", "abc,")); + assert!(!comma_separated_contains("avx2,fma", "avx")); +} + + diff --git a/src/dgemm_kernel.rs b/src/dgemm_kernel.rs index 1325eb0..0e1779a 100644 --- a/src/dgemm_kernel.rs +++ b/src/dgemm_kernel.rs @@ -29,15 +29,10 @@ use crate::packing::PackSlice; struct KernelAvx; #[cfg(any(target_arch="x86", target_arch="x86_64"))] struct KernelFmaAvx2; -#[cfg(any(target_arch="x86", target_arch="x86_64"))] -struct KernelFma; -#[cfg(any(target_arch="x86", target_arch="x86_64"))] -struct KernelSse2; #[cfg(has_avx512)] struct KernelAvx512; #[cfg(target_arch="aarch64")] -#[cfg(has_aarch64_simd)] struct KernelNeon; struct KernelFallback; @@ -60,20 +55,14 @@ pub(crate) fn detect(selector: G) where G: GemmSelect { return selector.select(KernelAvx512); } } - if is_x86_feature_detected_!("fma") { - if is_x86_feature_detected_!("avx2") { - return selector.select(KernelFmaAvx2); - } - return selector.select(KernelFma); + if is_x86_feature_detected_!("fma") && is_x86_feature_detected_!("avx2") { + return selector.select(KernelFmaAvx2); } else if is_x86_feature_detected_!("avx") { return selector.select(KernelAvx); - } else if is_x86_feature_detected_!("sse2") { - return selector.select(KernelSse2); } } #[cfg(target_arch="aarch64")] - #[cfg(has_aarch64_simd)] { if is_aarch64_feature_detected_!("neon") { return selector.select(KernelNeon); @@ -124,41 +113,6 @@ impl GemmKernel for KernelAvx { } } -#[cfg(any(target_arch="x86", target_arch="x86_64"))] -impl GemmKernel for KernelFma { - type Elem = T; - - type MRTy = ::MRTy; - type NRTy = ::NRTy; - - #[inline(always)] - fn align_to() -> usize { KernelAvx::align_to() } - - #[inline(always)] - fn always_masked() -> bool { KernelAvx::always_masked() } - - #[inline(always)] - fn nc() -> usize { archparam::D_NC } - #[inline(always)] - fn kc() -> usize { archparam::D_KC } - #[inline(always)] - fn mc() -> usize { archparam::D_MC } - - #[inline(always)] - unsafe fn kernel( - k: usize, - alpha: T, - a: *const T, - b: *const T, - beta: T, - c: *mut T, - rsc: isize, - csc: isize) - { - kernel_target_fma(k, alpha, a, b, beta, c, rsc, csc) - } -} - #[cfg(any(target_arch="x86", target_arch="x86_64"))] impl GemmKernel for KernelFmaAvx2 { type Elem = T; @@ -211,41 +165,6 @@ impl GemmKernel for KernelFmaAvx2 { } } -#[cfg(any(target_arch="x86", target_arch="x86_64"))] -impl GemmKernel for KernelSse2 { - type Elem = T; - - type MRTy = U4; - type NRTy = U4; - - #[inline(always)] - fn align_to() -> usize { 16 } - - #[inline(always)] - fn always_masked() -> bool { true } - - #[inline(always)] - fn nc() -> usize { archparam::D_NC } - #[inline(always)] - fn kc() -> usize { archparam::D_KC } - #[inline(always)] - fn mc() -> usize { archparam::D_MC } - - #[inline(always)] - unsafe fn kernel( - k: usize, - alpha: T, - a: *const T, - b: *const T, - beta: T, - c: *mut T, - rsc: isize, - csc: isize) - { - kernel_target_sse2(k, alpha, a, b, beta, c, rsc, csc) - } -} - #[cfg(has_avx512)] impl GemmKernel for KernelAvx512 { type Elem = T; @@ -295,7 +214,6 @@ impl GemmKernel for KernelAvx512 { } #[cfg(target_arch="aarch64")] -#[cfg(has_aarch64_simd)] impl GemmKernel for KernelNeon { type Elem = T; @@ -379,15 +297,6 @@ unsafe fn kernel_target_avx(k: usize, alpha: T, a: *const T, b: *const T, kernel_x86_avx::(k, alpha, a, b, beta, c, rsc, csc) } -#[inline] -#[target_feature(enable="sse2")] -#[cfg(any(target_arch="x86", target_arch="x86_64"))] -unsafe fn kernel_target_sse2(k: usize, alpha: T, a: *const T, b: *const T, - beta: T, c: *mut T, rsc: isize, csc: isize) -{ - kernel_fallback_impl(k, alpha, a, b, beta, c, rsc, csc) -} - #[inline(always)] #[cfg(any(target_arch="x86", target_arch="x86_64"))] unsafe fn kernel_x86_avx(k: usize, alpha: T, a: *const T, b: *const T, @@ -1006,7 +915,6 @@ unsafe fn kernel_target_avx512(k: usize, alpha: T, a: *const T, b: *const T, } #[cfg(target_arch="aarch64")] -#[cfg(has_aarch64_simd)] #[target_feature(enable="neon")] unsafe fn kernel_target_neon(k: usize, alpha: T, a: *const T, b: *const T, beta: T, c: *mut T, rsc: isize, csc: isize) @@ -1186,7 +1094,6 @@ mod tests { } #[cfg(any(target_arch="aarch64"))] - #[cfg(has_aarch64_simd)] mod test_kernel_aarch64 { use super::test_a_kernel; use super::super::*; @@ -1220,16 +1127,17 @@ mod tests { use super::super::*; #[cfg(feature = "std")] use std::println; + macro_rules! test_arch_kernels_x86 { - ($($feature_name:tt, $name:ident, $kernel_ty:ty),*) => { + ($([$($feature_name:tt),+], $name:ident, $kernel_ty:ty),*) => { $( #[test] fn $name() { - if is_x86_feature_detected_!($feature_name) { + if $(is_x86_feature_detected_!($feature_name) &&)+ true { test_a_kernel::<$kernel_ty, _>(stringify!($name)); } else { #[cfg(feature = "std")] - println!("Skipping, host does not have feature: {:?}", $feature_name); + println!("Skipping, host does not have feature(s): {:?}", &[$($feature_name),+]); } } )* @@ -1237,14 +1145,13 @@ mod tests { } test_arch_kernels_x86! { - "fma", fma, KernelFma, - "avx", avx, KernelAvx, - "sse2", sse2, KernelSse2 + ["fma", "avx2"], fma_avx2, KernelFmaAvx2, + ["avx"], avx, KernelAvx } #[cfg(has_avx512)] test_arch_kernels_x86! { - "avx512f", avx512f, KernelAvx512 + ["avx512f"], avx512f, KernelAvx512 } } } diff --git a/src/lib.rs b/src/lib.rs index 23dd24e..e55aa3e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -55,9 +55,8 @@ //! compile time, and the following kernel variants are //! implemented: //! -//! - `fma` //! - `avx` -//! - `sse2` +//! - `avx2,fma` //! - `avx512f` //! //! - *aarch64* features can be detected at runtime by default or compile time, @@ -130,14 +129,13 @@ //! The functions in this crate are thread safe, as long as the destination //! matrix is distinct. //! -//! ## Rust Version +//! ## Rust Version (MSRV) //! -//! This version requires Rust 1.41.1 or later; the crate follows a carefully +//! This version requires Rust 1.75 or later; the crate follows a carefully //! considered upgrade policy, where updating the minimum Rust version is not a breaking //! change. //! -//! Some features are enabled with later versions: from Rust 1.61 AArch64 NEON -//! support, and from Rust 1.89 x86/x86-64 AVX-512 support. +//! Some features are enabled with later versions: from Rust 1.89 x86/x86-64 AVX-512 support. #![doc(html_root_url = "https://docs.rs/matrixmultiply/0.3/")] #![cfg_attr(not(feature = "std"), no_std)] @@ -172,15 +170,20 @@ mod threading; mod aligned_alloc; mod util; -#[macro_use] -mod archmacros; +#[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))] +mod constfind; +#[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))] +mod target_features; +#[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))] +pub(crate) use crate::target_features::allow_feature; #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] #[macro_use] mod x86; -#[cfg(any(target_arch = "aarch64"))] +#[cfg(target_arch = "aarch64")] #[macro_use] mod aarch64; + mod dgemm_kernel; mod sgemm_kernel; diff --git a/src/sgemm_kernel.rs b/src/sgemm_kernel.rs index 26a0c4d..6b27cda 100644 --- a/src/sgemm_kernel.rs +++ b/src/sgemm_kernel.rs @@ -30,15 +30,10 @@ use crate::packing::PackSlice; struct KernelAvx; #[cfg(any(target_arch="x86", target_arch="x86_64"))] struct KernelFmaAvx2; -#[cfg(any(target_arch="x86", target_arch="x86_64"))] -struct KernelFma; -#[cfg(any(target_arch="x86", target_arch="x86_64"))] -struct KernelSse2; #[cfg(has_avx512)] struct KernelAvx512; #[cfg(target_arch="aarch64")] -#[cfg(has_aarch64_simd)] struct KernelNeon; #[cfg(all(target_arch="wasm32", target_feature="simd128"))] struct KernelWasmSimd; @@ -62,19 +57,13 @@ pub(crate) fn detect(selector: G) where G: GemmSelect { return selector.select(KernelAvx512); } } - if is_x86_feature_detected_!("fma") { - if is_x86_feature_detected_!("avx2") { - return selector.select(KernelFmaAvx2); - } - return selector.select(KernelFma); + if is_x86_feature_detected_!("fma") && is_x86_feature_detected_!("avx2") { + return selector.select(KernelFmaAvx2); } else if is_x86_feature_detected_!("avx") { return selector.select(KernelAvx); - } else if is_x86_feature_detected_!("sse2") { - return selector.select(KernelSse2); } } #[cfg(target_arch="aarch64")] - #[cfg(has_aarch64_simd)] { if is_aarch64_feature_detected_!("neon") { return selector.select(KernelNeon); @@ -125,38 +114,6 @@ impl GemmKernel for KernelAvx { } } -#[cfg(any(target_arch="x86", target_arch="x86_64"))] -impl GemmKernel for KernelFma { - type Elem = T; - - type MRTy = ::MRTy; - type NRTy = ::NRTy; - - #[inline(always)] - fn align_to() -> usize { KernelAvx::align_to() } - - #[inline(always)] - fn always_masked() -> bool { KernelAvx::always_masked() } - - #[inline(always)] - fn nc() -> usize { archparam::S_NC } - #[inline(always)] - fn kc() -> usize { archparam::S_KC } - #[inline(always)] - fn mc() -> usize { archparam::S_MC } - - #[inline(always)] - unsafe fn kernel( - k: usize, - alpha: T, - a: *const T, - b: *const T, - beta: T, - c: *mut T, rsc: isize, csc: isize) { - kernel_target_fma(k, alpha, a, b, beta, c, rsc, csc) - } -} - #[cfg(any(target_arch="x86", target_arch="x86_64"))] impl GemmKernel for KernelFmaAvx2 { type Elem = T; @@ -205,38 +162,6 @@ impl GemmKernel for KernelFmaAvx2 { } } -#[cfg(any(target_arch="x86", target_arch="x86_64"))] -impl GemmKernel for KernelSse2 { - type Elem = T; - - type MRTy = ::MRTy; - type NRTy = ::NRTy; - - #[inline(always)] - fn align_to() -> usize { 16 } - - #[inline(always)] - fn always_masked() -> bool { KernelFallback::always_masked() } - - #[inline(always)] - fn nc() -> usize { archparam::S_NC } - #[inline(always)] - fn kc() -> usize { archparam::S_KC } - #[inline(always)] - fn mc() -> usize { archparam::S_MC } - - #[inline(always)] - unsafe fn kernel( - k: usize, - alpha: T, - a: *const T, - b: *const T, - beta: T, - c: *mut T, rsc: isize, csc: isize) { - kernel_target_sse2(k, alpha, a, b, beta, c, rsc, csc) - } -} - #[cfg(has_avx512)] impl GemmKernel for KernelAvx512 { type Elem = T; @@ -286,7 +211,6 @@ impl GemmKernel for KernelAvx512 { } #[cfg(target_arch="aarch64")] -#[cfg(has_aarch64_simd)] impl GemmKernel for KernelNeon { type Elem = T; @@ -399,15 +323,6 @@ unsafe fn kernel_target_avx(k: usize, alpha: T, a: *const T, b: *const T, kernel_x86_avx::(k, alpha, a, b, beta, c, rsc, csc) } -#[inline] -#[cfg(any(target_arch="x86", target_arch="x86_64"))] -#[target_feature(enable="sse2")] -unsafe fn kernel_target_sse2(k: usize, alpha: T, a: *const T, b: *const T, - beta: T, c: *mut T, rsc: isize, csc: isize) -{ - kernel_fallback_impl(k, alpha, a, b, beta, c, rsc, csc) -} - #[inline(always)] #[cfg(any(target_arch="x86", target_arch="x86_64"))] unsafe fn kernel_x86_avx(k: usize, alpha: T, a: *const T, b: *const T, @@ -661,7 +576,6 @@ unsafe fn kernel_target_avx512(k: usize, alpha: T, a: *const T, b: *const T, } #[cfg(target_arch="aarch64")] -#[cfg(has_aarch64_simd)] #[target_feature(enable="neon")] unsafe fn kernel_target_neon(k: usize, alpha: T, a: *const T, b: *const T, beta: T, c: *mut T, rsc: isize, csc: isize) @@ -994,7 +908,6 @@ mod tests { } #[cfg(any(target_arch="aarch64"))] - #[cfg(has_aarch64_simd)] mod test_kernel_aarch64 { use super::test_a_kernel; use super::super::*; @@ -1041,15 +954,15 @@ mod tests { use std::println; macro_rules! test_arch_kernels_x86 { - ($($feature_name:tt, $name:ident, $kernel_ty:ty),*) => { + ($([$($feature_name:tt),+], $name:ident, $kernel_ty:ty),*) => { $( #[test] fn $name() { - if is_x86_feature_detected_!($feature_name) { + if $(is_x86_feature_detected_!($feature_name) &&)+ true { test_a_kernel::<$kernel_ty, _>(stringify!($name)); } else { #[cfg(feature = "std")] - println!("Skipping, host does not have feature: {:?}", $feature_name); + println!("Skipping, host does not have feature(s): {:?}", &[$($feature_name),+]); } } )* @@ -1057,14 +970,13 @@ mod tests { } test_arch_kernels_x86! { - "fma", fma, KernelFma, - "avx", avx, KernelAvx, - "sse2", sse2, KernelSse2 + ["fma", "avx2"], fma_avx2, KernelFmaAvx2, + ["avx"], avx, KernelAvx } #[cfg(has_avx512)] test_arch_kernels_x86! { - "avx512f", avx512f, KernelAvx512 + ["avx512f"], avx512f, KernelAvx512 } #[test] @@ -1077,17 +989,19 @@ mod tests { // skip return; } - let feature_name = option_env!("MMTEST_FEATURE") + let feature_names = option_env!("MMTEST_FEATURE") .expect("No MMTEST_FEATURE configured!"); - let detected = match feature_name { - "avx" => is_x86_feature_detected_!("avx"), - "fma" => is_x86_feature_detected_!("fma"), - "sse2" => is_x86_feature_detected_!("sse2"), - "avx512f" => is_x86_feature_detected_!("avx512f"), - _ => false, - }; - assert!(detected, "Feature {:?} was not detected, so it could not be tested", - feature_name); + for feature_name in feature_names.split(",") { + let detected = match feature_name { + "avx" => is_x86_feature_detected_!("avx"), + "fma" => is_x86_feature_detected_!("fma"), + "avx2" => is_x86_feature_detected_!("avx2"), + "avx512f" => is_x86_feature_detected_!("avx512f"), + _ => panic!("Unknown feature {:?}", feature_name), + }; + assert!(detected, "Feature {:?} was not detected, so it could not be tested", + feature_name); + } } } } diff --git a/src/target_features.rs b/src/target_features.rs new file mode 100644 index 0000000..83f915d --- /dev/null +++ b/src/target_features.rs @@ -0,0 +1,30 @@ + +/// Is the target feature allowed currently? +/// +/// The environment variable MMTEST_FEATURE is read at compile-time. +/// Used for testing only - if the environment variable is non-empty, +/// **only** features listed, comma-separated, are allowed to be detected, +/// all other are disabled. +/// +/// This is internal only, not stable. +pub(crate) const fn allow_feature(feature: &str) -> bool { + match option_env!("MMTEST_FEATURE") { + None => true, + Some(s) if s.is_empty() => true, + Some(value) => crate::constfind::comma_separated_contains(value, feature), + } +} + + + +#[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch="aarch64"))] +#[test] +fn test_features() { + // This is just a test you can run to see the effect + // of environment variable parsing. + let features = &["sse2", "avx", "avx2", "fma", "avx512f", "neon"]; + for feat in features { + println!(r#"feature= {:12} allowed= {}"#, feat, allow_feature(feat)); + + } +} diff --git a/src/x86/macros.rs b/src/x86/macros.rs index 612f202..a2ca653 100644 --- a/src/x86/macros.rs +++ b/src/x86/macros.rs @@ -5,7 +5,7 @@ macro_rules! is_x86_feature_detected_ { // For testing purposes, we can make sure only one specific feature // is enabled by setting MMTEST_FEATURE=featurename (all others // disabled). This does not force it to be detected, it must also be. - compile_env_matches_or_is_empty!("MMTEST_FEATURE", $name) && is_x86_feature_detected!($name) + crate::allow_feature($name) && is_x86_feature_detected!($name) } #[cfg(not(feature="std"))] { @@ -15,7 +15,7 @@ macro_rules! is_x86_feature_detected_ { // be. In the `no_std` case, the `is_86_feature_detected` macro is // not available, so we have to fall back to checking whether the // feature is enabled at compile-time. - compile_env_matches_or_is_empty!("MMTEST_FEATURE", $name) && cfg!(target_feature=$name) + crate::allow_feature($name) && cfg!(target_feature=$name) } }}; } diff --git a/src/zgemm_kernel.rs b/src/zgemm_kernel.rs index 71f4ce6..26df3b9 100644 --- a/src/zgemm_kernel.rs +++ b/src/zgemm_kernel.rs @@ -18,11 +18,8 @@ use crate::packing::PackSlice; struct KernelAvx512; #[cfg(any(target_arch="x86", target_arch="x86_64"))] struct KernelAvx2; -#[cfg(any(target_arch="x86", target_arch="x86_64"))] -struct KernelFma; #[cfg(target_arch = "aarch64")] -#[cfg(has_aarch64_simd)] struct KernelNeon; struct KernelFallback; @@ -46,15 +43,11 @@ pub(crate) fn detect(selector: G) where G: GemmSelect { return selector.select(KernelAvx512); } } - if is_x86_feature_detected_!("fma") { - if is_x86_feature_detected_!("avx2") { - return selector.select(KernelAvx2); - } - return selector.select(KernelFma); + if is_x86_feature_detected_!("fma") && is_x86_feature_detected_!("avx2") { + return selector.select(KernelAvx2); } } #[cfg(target_arch = "aarch64")] - #[cfg(has_aarch64_simd)] { if is_aarch64_feature_detected_!("neon") { return selector.select(KernelNeon); @@ -134,42 +127,7 @@ impl GemmKernel for KernelAvx2 { } } -#[cfg(any(target_arch="x86", target_arch="x86_64"))] -impl GemmKernel for KernelFma { - type Elem = T; - - type MRTy = ::MRTy; - type NRTy = ::NRTy; - - #[inline(always)] - fn align_to() -> usize { 16 } - - #[inline(always)] - fn always_masked() -> bool { KernelFallback::always_masked() } - - #[inline(always)] - fn nc() -> usize { archparam::Z_NC } - #[inline(always)] - fn kc() -> usize { archparam::Z_KC } - #[inline(always)] - fn mc() -> usize { archparam::Z_MC } - - pack_methods!{} - - #[inline(always)] - unsafe fn kernel( - k: usize, - alpha: T, - a: *const T, - b: *const T, - beta: T, - c: *mut T, rsc: isize, csc: isize) { - kernel_target_fma(k, alpha, a, b, beta, c, rsc, csc) - } -} - #[cfg(target_arch = "aarch64")] -#[cfg(has_aarch64_simd)] impl GemmKernel for KernelNeon { type Elem = T; @@ -239,21 +197,13 @@ impl GemmKernel for KernelFallback { #[cfg(any(target_arch="x86", target_arch="x86_64"))] kernel_fallback_impl_complex! { // instantiate fma separately - [inline target_feature(enable="fma") target_feature(enable="avx2")] [fma_yes] + [inline target_feature(enable="fma,avx2")] [fma_yes] kernel_target_avx2, T, TReal, KernelAvx2::MR, KernelAvx2::NR, 4 } -#[cfg(any(target_arch="x86", target_arch="x86_64"))] -kernel_fallback_impl_complex! { - // instantiate fma separately - [inline target_feature(enable="fma")] [fma_no] - kernel_target_fma, T, TReal, KernelFma::MR, KernelFma::NR, 2 -} - // Kernel neon #[cfg(target_arch = "aarch64")] -#[cfg(has_aarch64_simd)] kernel_fallback_impl_complex! { [inline target_feature(enable="neon")] [fma_yes] kernel_target_neon, T, TReal, KernelNeon::MR, KernelNeon::NR, 1 @@ -290,7 +240,6 @@ mod tests { } #[cfg(target_arch = "aarch64")] - #[cfg(has_aarch64_simd)] mod test_kernel_aarch64 { use super::test_complex_packed_kernel; use super::super::*; @@ -340,7 +289,6 @@ mod tests { } test_arch_kernels_x86! { - "fma", fma, KernelFma, "avx2", avx2, KernelAvx2 }