diff --git a/AGENTS.md b/AGENTS.md index e73ae2bd6..1cf4774f2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -108,6 +108,7 @@ cmake --build build | `ADA_BUILD_SINGLE_HEADER_LIB` | OFF | Build from single-header amalgamated files | | `ADA_USE_SIMDUTF` | OFF | Enable SIMD-accelerated Unicode via simdutf | | `ADA_USE_UNSAFE_STD_REGEX_PROVIDER` | OFF (ON when `ADA_TESTING=ON`) | Enable `std_regex_provider` (`std::regex`-backed). Required to build the `urlpattern` benchmark or any code using `ada::url_pattern_regex::std_regex_provider`. Not recommended for production (ReDoS risk) | +| `ADA_X86_64_V2` | OFF | Compile the library with `-march=x86-64-v2` (MSVC `/arch:SSE4.2`). Platforms that already default to that baseline (Rocky/RHEL 9) and MSVC `/arch:SSE4.2` or AVX builds enable the SSSE3 kernels without this option | | `CMAKE_BUILD_TYPE` | - | Set to `Release` for optimized builds, `Debug` for development | ## Running Tests diff --git a/README.md b/README.md index abbbbd6b2..4f19b5813 100644 --- a/README.md +++ b/README.md @@ -367,6 +367,7 @@ With tests (requires available local packages): Ada provides several CMake options to customize the build: - `ADA_USE_SIMDUTF`: Enables SIMD-accelerated Unicode processing via simdutf (default: OFF) +- `ADA_X86_64_V2`: Compile with `-march=x86-64-v2` (MSVC `/arch:SSE4.2`) for SSSE3, SSE4.1, SSE4.2, and POPCNT. Platforms that already default to that baseline (Rocky Linux 9, RHEL 9) enable those kernels without this option. MSVC builds that request AVX or `/arch:SSE4.2` also enable them. Windows users need additional flags to specify the build configuration, e.g. `--config Release`. diff --git a/cmake/ada-flags.cmake b/cmake/ada-flags.cmake index cc59e3b2b..291f07914 100644 --- a/cmake/ada-flags.cmake +++ b/cmake/ada-flags.cmake @@ -20,6 +20,9 @@ option(ADA_BENCHMARKS "Build benchmarks" OFF) option(ADA_TESTING "Build tests" OFF) option(ADA_USE_UNSAFE_STD_REGEX_PROVIDER "Enable unsafe regex provider that uses std::regex" OFF) option(ADA_INCLUDE_URL_PATTERN "Include URL pattern implementation" ON) +option(ADA_X86_64_V2 + "Target x86-64-v2 (SSSE3, SSE4.1, SSE4.2, POPCNT) for the whole library via -march=x86-64-v2 (MSVC /arch:SSE4.2). Platforms that already default to that baseline (Rocky/RHEL 9) enable the kernels without this option." + OFF) if (ADA_COVERAGE) message(STATUS "You want to compute coverage. We assume that you have installed gcovr.") diff --git a/include/ada/common_defs.h b/include/ada/common_defs.h index e773cb34d..b53c2b552 100644 --- a/include/ada/common_defs.h +++ b/include/ada/common_defs.h @@ -241,12 +241,37 @@ namespace ada { #define ADA_SSSE3 1 #endif +#if defined(__SSE4_1__) +#define ADA_SSE41 1 +#endif + +#if defined(__SSE4_2__) +#define ADA_SSE42 1 +#endif + +#if defined(__POPCNT__) +#define ADA_POPCNT 1 +#endif + #if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \ (defined(_M_AMD64) || defined(_M_X64) || \ (defined(_M_IX86_FP) && _M_IX86_FP == 2)) #define ADA_SSE2 1 #endif +// x86-64-v2 = SSE2 + SSE3 + SSSE3 + SSE4.1 + SSE4.2 + POPCNT. +// Windows 11 24H2, Rocky Linux 9, and RHEL 9 default to this baseline. +// MSVC does not define the GNU __SSSE3__/__SSE4_*__ macros unless /arch +// requests them; AVX (and the newer /arch:SSE4.2) imply the v2 ISA. +#if !defined(ADA_SSSE3) && defined(ADA_SSE2) && defined(_MSC_VER) && \ + !defined(__clang__) && \ + (defined(__AVX__) || defined(__AVX2__) || defined(__SSE4_2__)) +#define ADA_SSSE3 1 +#define ADA_SSE41 1 +#define ADA_SSE42 1 +#define ADA_POPCNT 1 +#endif + // AVX-512 byte/word ops + 128/256-bit vectors of AVX-512 instructions. // Used for optional high-performance IP address parsing kernels. #if defined(__AVX512BW__) && defined(__AVX512VL__) diff --git a/include/ada/unicode.h b/include/ada/unicode.h index df2e11380..a95ef4229 100644 --- a/include/ada/unicode.h +++ b/include/ada/unicode.h @@ -71,7 +71,9 @@ bool to_ascii(std::optional& out, std::string_view plain, * Checks if the input has tab or newline characters. * * @attention The has_tabs_or_newline function is a bottleneck and it is simple - * enough that compilers like GCC can 'autovectorize it'. + * enough that compilers like GCC can 'autovectorize it'. On x86-64-v2 + * (__SSSE3__) it uses an SSSE3 pshufb table; SSE4.1 adds ptest on the miss + * path. */ ada_really_inline bool has_tabs_or_newline( std::string_view user_input) noexcept; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c1a6e1d2c..8cb38e6a9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -38,6 +38,18 @@ endif(MSVC) if ((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (CMAKE_SYSTEM_PROCESSOR MATCHES "^(i.86|x86(_64)?)$")) target_compile_options(ada PRIVATE -mno-avx256-split-unaligned-load -mno-avx256-split-unaligned-store) endif() + +# Optional whole-library x86-64-v2 target. When the compiler already +# defines __SSSE3__ (Rocky/RHEL 9, -march=x86-64-v2), the SSSE3 kernels +# in unicode.cpp/helpers.cpp compile without this option. +if(ADA_X86_64_V2 AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|amd64)$") + if(MSVC) + target_compile_options(ada PRIVATE /arch:SSE4.2) + else() + target_compile_options(ada PRIVATE -march=x86-64-v2) + endif() + message(STATUS "Targeting x86-64-v2 (SSSE3, SSE4.1, SSE4.2, POPCNT)") +endif() if(ADA_DEVELOPMENT_CHECKS) target_compile_definitions(ada PUBLIC ADA_DEVELOPMENT_CHECKS=1) endif() diff --git a/src/helpers.cpp b/src/helpers.cpp index a66e8dca8..016f36c2d 100644 --- a/src/helpers.cpp +++ b/src/helpers.cpp @@ -1029,6 +1029,102 @@ ada_really_inline void strip_trailing_spaces_from_opaque_path(url_type& url) { url.update_base_pathname(path); } +#if ADA_SSSE3 +ada_really_inline size_t +find_authority_delimiter_special(std::string_view view) noexcept { + if (view.size() < 16) { + for (size_t i = 0; i < view.size(); i++) { + if (view[i] == '@' || view[i] == '/' || view[i] == '\\' || + view[i] == '?') { + return i; + } + } + return view.size(); + } + size_t i = 0; + const __m128i low_mask = + _mm_setr_epi8(0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x08, 0x00, 0x00, 0x06); + const __m128i high_mask = + _mm_setr_epi8(0x00, 0x00, 0x02, 0x04, 0x01, 0x08, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + const __m128i fmask = _mm_set1_epi8(0xf); + const __m128i zero = _mm_setzero_si128(); + for (; i + 15 < view.size(); i += 16) { + const __m128i word = _mm_loadu_si128((const __m128i*)(view.data() + i)); + const __m128i classify = _mm_and_si128( + _mm_shuffle_epi8(low_mask, _mm_and_si128(word, fmask)), + _mm_shuffle_epi8(high_mask, + _mm_and_si128(_mm_srli_epi16(word, 4), fmask))); + const int mask = + ~_mm_movemask_epi8(_mm_cmpeq_epi8(classify, zero)) & 0xFFFF; + if (mask != 0) { + return i + trailing_zeroes(static_cast(mask)); + } + } + if (i < view.size()) { + const __m128i word = + _mm_loadu_si128((const __m128i*)(view.data() + view.length() - 16)); + const __m128i classify = _mm_and_si128( + _mm_shuffle_epi8(low_mask, _mm_and_si128(word, fmask)), + _mm_shuffle_epi8(high_mask, + _mm_and_si128(_mm_srli_epi16(word, 4), fmask))); + const int mask = + ~_mm_movemask_epi8(_mm_cmpeq_epi8(classify, zero)) & 0xFFFF; + if (mask != 0) { + return view.length() - 16 + trailing_zeroes(static_cast(mask)); + } + } + return view.size(); +} + +ada_really_inline size_t +find_authority_delimiter(std::string_view view) noexcept { + if (view.size() < 16) { + for (size_t i = 0; i < view.size(); i++) { + if (view[i] == '@' || view[i] == '/' || view[i] == '?') { + return i; + } + } + return view.size(); + } + size_t i = 0; + const __m128i low_mask = + _mm_setr_epi8(0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x06); + const __m128i high_mask = + _mm_setr_epi8(0x00, 0x00, 0x02, 0x04, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00); + const __m128i fmask = _mm_set1_epi8(0xf); + const __m128i zero = _mm_setzero_si128(); + for (; i + 15 < view.size(); i += 16) { + const __m128i word = _mm_loadu_si128((const __m128i*)(view.data() + i)); + const __m128i classify = _mm_and_si128( + _mm_shuffle_epi8(low_mask, _mm_and_si128(word, fmask)), + _mm_shuffle_epi8(high_mask, + _mm_and_si128(_mm_srli_epi16(word, 4), fmask))); + const int mask = + ~_mm_movemask_epi8(_mm_cmpeq_epi8(classify, zero)) & 0xFFFF; + if (mask != 0) { + return i + trailing_zeroes(static_cast(mask)); + } + } + if (i < view.size()) { + const __m128i word = + _mm_loadu_si128((const __m128i*)(view.data() + view.length() - 16)); + const __m128i classify = _mm_and_si128( + _mm_shuffle_epi8(low_mask, _mm_and_si128(word, fmask)), + _mm_shuffle_epi8(high_mask, + _mm_and_si128(_mm_srli_epi16(word, 4), fmask))); + const int mask = + ~_mm_movemask_epi8(_mm_cmpeq_epi8(classify, zero)) & 0xFFFF; + if (mask != 0) { + return view.length() - 16 + trailing_zeroes(static_cast(mask)); + } + } + return view.size(); +} +#else // @ / \\ ? static constexpr std::array authority_delimiter_special = []() consteval { @@ -1071,6 +1167,7 @@ find_authority_delimiter(std::string_view view) noexcept { } return size_t(view.size()); } +#endif } // namespace ada::helpers diff --git a/src/unicode.cpp b/src/unicode.cpp index 6d1f5a24e..e48a09b6f 100644 --- a/src/unicode.cpp +++ b/src/unicode.cpp @@ -14,6 +14,9 @@ ADA_POP_DISABLE_WARNINGS #include #if ADA_SSSE3 #include +#if ADA_SSE41 +#include +#endif #elif ADA_NEON #include #elif ADA_SSE2 @@ -61,6 +64,7 @@ constexpr bool to_lower_ascii(char* input, size_t length) noexcept { } return non_ascii == 0; } + #if ADA_SSSE3 ada_really_inline bool has_tabs_or_newline( std::string_view user_input) noexcept { @@ -91,7 +95,12 @@ ada_really_inline bool has_tabs_or_newline( __m128i matches = _mm_cmpeq_epi8(shuffled, word); running = _mm_or_si128(running, matches); } +#if ADA_SSE41 + // SSE4.1 ptest avoids a movemask + GPR transfer on the common miss path. + return _mm_testz_si128(running, running) == 0; +#else return _mm_movemask_epi8(running) != 0; +#endif } #elif ADA_NEON ada_really_inline bool has_tabs_or_newline( @@ -244,7 +253,6 @@ ada_really_inline bool has_tabs_or_newline( return running; } #endif - // A forbidden host code point is U+0000 NULL, U+0009 TAB, U+000A LF, U+000D CR, // U+0020 SPACE, U+0023 (#), U+002F (/), U+003A (:), U+003C (<), U+003E (>), // U+003F (?), U+0040 (@), U+005B ([), U+005C (\), U+005D (]), U+005E (^), or diff --git a/tests/basic_tests.cpp b/tests/basic_tests.cpp index 5ce551db9..0a236909e 100644 --- a/tests/basic_tests.cpp +++ b/tests/basic_tests.cpp @@ -2,6 +2,7 @@ #include "gtest/gtest.h" #include #include +#include using Types = testing::Types; template @@ -2294,3 +2295,56 @@ TYPED_TEST(basic_tests, // a byte that legitimately needs encoding is still encoded check("http://ab?x y", "http://ab/?x%20y"); } + +TYPED_TEST(basic_tests, x86_64_v2_long_url_parse) { + auto r = ada::parse( + "https://abcdefghijklmnopqrstuvwxyz012345.example.com/path"); + ASSERT_TRUE(r); + ASSERT_EQ(r->get_hostname(), "abcdefghijklmnopqrstuvwxyz012345.example.com"); + + r = ada::parse( + "https://user:password@abcdefghijklmnopqrstuvwxyz.example.com/foo"); + ASSERT_TRUE(r); + ASSERT_EQ(r->get_username(), "user"); + ASSERT_EQ(r->get_password(), "password"); + ASSERT_EQ(r->get_hostname(), "abcdefghijklmnopqrstuvwxyz.example.com"); + + // Long userinfo so the authority delimiter sits past a 16-byte window. + const std::string long_user(40, 'u'); + r = ada::parse("https://" + long_user + "@example.com/foo"); + ASSERT_TRUE(r); + ASSERT_EQ(r->get_username(), long_user); + ASSERT_EQ(r->get_hostname(), "example.com"); + ASSERT_EQ(r->get_pathname(), "/foo"); + + r = ada::parse("https://[::1]:8080/path"); + ASSERT_TRUE(r); + ASSERT_EQ(r->get_hostname(), "[::1]"); + ASSERT_EQ(r->get_port(), "8080"); + ASSERT_EQ(r->get_pathname(), "/path"); + + // Tabs/newlines must be stripped on a string long enough for the SSSE3 + // window, including a match that sits only in the overlapping tail. + std::string with_tab = "https://abcdefghijklmnopqrstuvwxyz.example.com/foo"; + with_tab.insert(with_tab.size() - 2, "\t"); + r = ada::parse(with_tab); + ASSERT_TRUE(r); + ASSERT_EQ(r->get_pathname(), "/foo"); + + const std::string long_host = "abcdefghijklmnopqrstuvwxyz.example.com"; + const size_t tab_positions[] = {0, 8, 16, long_host.size() - 1}; + for (char ws : {'\t', '\n', '\r'}) { + for (size_t pos : tab_positions) { + std::string host = long_host; + host.insert(pos, 1, ws); + r = ada::parse("https://" + host + "/"); + ASSERT_TRUE(r) << "ws=" << int(ws) << " pos=" << pos; + ASSERT_EQ(r->get_hostname(), long_host); + } + } + + r = ada::parse("https://example.com/#newfragment"); + ASSERT_TRUE(r); + r->set_hash("#otherfragment"); + ASSERT_EQ(r->get_hash(), "#otherfragment"); +}