diff --git a/benchmarks/percent_encode.cpp b/benchmarks/percent_encode.cpp index c88e50655..5f31f5692 100644 --- a/benchmarks/percent_encode.cpp +++ b/benchmarks/percent_encode.cpp @@ -243,6 +243,47 @@ static void C0Control(benchmark::State& state) { } BENCHMARK(C0Control); +std::string long_examples[] = { + "connect timeout=10 application name=myapp server=db host internal " + "database=production analytics read preference=secondary preferred " + "ssl=true retry writes=true w=majority max pool size=50", + "ref=web twc ao gbl adsinfo utm source=twc utm medium=cpc " + "utm campaign=brand awareness q4 2024 utm content=banner 300x250 " + "utm term=weather forecast today gclid=Cj0KCQiA3Y ABhCnARIsAK", +}; + +double long_examples_bytes = []() -> double { + size_t bytes{0}; + for (std::string& url_string : long_examples) { + bytes += url_string.size(); + } + return double(bytes); +}(); + +static void LongFragment(benchmark::State& state) { + for (auto _ : state) { + for (std::string& url_string : long_examples) { + benchmark::DoNotOptimize(ada::unicode::percent_encode( + url_string, ada::character_sets::FRAGMENT_PERCENT_ENCODE)); + } + } + state.counters["speed"] = benchmark::Counter( + long_examples_bytes, benchmark::Counter::kIsIterationInvariantRate); +} +BENCHMARK(LongFragment); + +static void LongQuery(benchmark::State& state) { + for (auto _ : state) { + for (std::string& url_string : long_examples) { + benchmark::DoNotOptimize(ada::unicode::percent_encode( + url_string, ada::character_sets::QUERY_PERCENT_ENCODE)); + } + } + state.counters["speed"] = benchmark::Counter( + long_examples_bytes, benchmark::Counter::kIsIterationInvariantRate); +} +BENCHMARK(LongQuery); + int main(int argc, char** argv) { #if defined(ADA_RUST_VERSION) benchmark::AddCustomContext("rust version ", ADA_RUST_VERSION); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9587ee089..c1a6e1d2c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -7,7 +7,7 @@ target_include_directories(ada-include-source INTERFACE $/ada.cpp) target_link_libraries(ada-source INTERFACE ada-include-source) -add_library(ada ada.cpp) +add_library(ada ada.cpp unicode_percent_encode.cpp) target_compile_features(ada PUBLIC cxx_std_20) target_include_directories(ada PRIVATE $ ) target_include_directories(ada PUBLIC "$") @@ -60,6 +60,10 @@ if(ADA_TESTING) target_compile_definitions(ada PRIVATE ADA_TESTING=1) endif() +# Keep the SIMD percent-encode kernel in its own TU so the unity ada.cpp +# inlining budget for setters matches main. +target_compile_definitions(ada PRIVATE ADA_PERCENT_ENCODE_SIMD_SEPARATE_TU=1) + if(ADA_INCLUDE_URL_PATTERN) target_compile_definitions(ada PRIVATE ADA_INCLUDE_URL_PATTERN=1) else() diff --git a/src/ada.cpp b/src/ada.cpp index 321dbef16..33ed25b2b 100644 --- a/src/ada.cpp +++ b/src/ada.cpp @@ -1,6 +1,9 @@ #include "ada.h" #include "checkers.cpp" #include "unicode.cpp" +#if !defined(ADA_PERCENT_ENCODE_SIMD_SEPARATE_TU) +#include "unicode_percent_encode.cpp" +#endif #include "serializers.cpp" #include "implementation.cpp" #include "helpers.cpp" diff --git a/src/unicode.cpp b/src/unicode.cpp index 7389fc983..6d1f5a24e 100644 --- a/src/unicode.cpp +++ b/src/unicode.cpp @@ -590,6 +590,9 @@ std::string form_urlencoded_decode(const std::string_view input) { return out; } +void percent_encode_suffix(const char* p, const char* end, + const uint8_t character_set[], std::string& out); + std::string percent_encode(const std::string_view input, const uint8_t character_set[]) { auto pointer = std::ranges::find_if(input, [character_set](const char c) { @@ -604,15 +607,18 @@ std::string percent_encode(const std::string_view input, result.reserve(input.length()); // in the worst case, percent encoding might // produce 3 characters. result.append(input.substr(0, std::distance(input.begin(), pointer))); - - for (; pointer != input.end(); pointer++) { - if (character_sets::bit_at(character_set, *pointer)) { - result.append(character_sets::hex + uint8_t(*pointer) * 4, 3); - } else { - result += *pointer; + if (static_cast(input.end() - pointer) >= 48) { + percent_encode_suffix(&*pointer, input.data() + input.size(), character_set, + result); + } else { + for (; pointer != input.end(); pointer++) { + if (character_sets::bit_at(character_set, *pointer)) { + result.append(character_sets::hex + uint8_t(*pointer) * 4, 3); + } else { + result += *pointer; + } } } - return result; } @@ -651,6 +657,13 @@ bool percent_encode(const std::string_view input, const uint8_t character_set[], return true; } +template bool percent_encode(std::string_view input, + const uint8_t character_set[], + std::string& out); +template bool percent_encode(std::string_view input, + const uint8_t character_set[], + std::string& out); + bool to_ascii(std::optional& out, const std::string_view plain, size_t first_percent) { std::string percent_decoded_buffer; @@ -675,11 +688,16 @@ std::string percent_encode(const std::string_view input, // NOLINTNEXTLINE(bugprone-suspicious-stringview-data-usage) out.append(input.data(), index); auto pointer = input.begin() + index; - for (; pointer != input.end(); pointer++) { - if (character_sets::bit_at(character_set, *pointer)) { - out.append(character_sets::hex + uint8_t(*pointer) * 4, 3); - } else { - out += *pointer; + if (static_cast(input.end() - pointer) >= 48) { + percent_encode_suffix(&*pointer, input.data() + input.size(), character_set, + out); + } else { + for (; pointer != input.end(); pointer++) { + if (character_sets::bit_at(character_set, *pointer)) { + out.append(character_sets::hex + uint8_t(*pointer) * 4, 3); + } else { + out += *pointer; + } } } return out; diff --git a/src/unicode_percent_encode.cpp b/src/unicode_percent_encode.cpp new file mode 100644 index 000000000..718095774 --- /dev/null +++ b/src/unicode_percent_encode.cpp @@ -0,0 +1,303 @@ +#include + +#include "ada/unicode.h" + +#include "ada/character_sets-inl.h" +#include "ada/character_sets.h" +#include "ada/common_defs.h" + +#include +#if ADA_SSSE3 +#include +#define ADA_UNICODE_SSSE3 1 +#elif ADA_NEON +#include +#elif ADA_RVV +#include +#endif + +// gcc/clang honor target("ssse3") on an SSE2 translation unit. clang-cl and +// MSVC do not: they still compile the function as SSE2, then reject +// always_inline _mm_shuffle_epi8. Same approach as parser.cpp. +#if !ADA_UNICODE_SSSE3 && (defined(__x86_64__) || defined(__amd64__)) && \ + defined(__GNUC__) && !defined(_MSC_VER) +#include +#define ADA_UNICODE_SSSE3 1 +#define ADA_UNICODE_NEED_SSSE3_TARGET 1 +#endif +#ifndef ADA_UNICODE_SSSE3 +#define ADA_UNICODE_SSSE3 0 +#endif +#ifdef ADA_UNICODE_NEED_SSSE3_TARGET +#define ADA_UNICODE_SIMD __attribute__((target("ssse3"))) +#else +#define ADA_UNICODE_SIMD ada_really_inline +#endif + +#ifdef ADA_REGULAR_VISUAL_STUDIO +#include +#endif + +namespace ada::unicode { +namespace { + +#if ADA_UNICODE_SSSE3 || ADA_NEON +ada_really_inline int trailing_zeroes32(uint32_t input_num) noexcept { +#ifdef ADA_REGULAR_VISUAL_STUDIO + unsigned long ret; + _BitScanForward(&ret, input_num); + return static_cast(ret); +#else + return __builtin_ctz(input_num); +#endif +} + +// Append one window whose set bits mark bytes that need encoding. +ada_really_inline void encode_mask_window(const char* p, uint32_t mask, + size_t width, std::string& out) { + uint64_t bits = mask; + size_t off = 0; + while (bits != 0) { + const int zero_run = trailing_zeroes32(static_cast(bits)); + if (zero_run != 0) { + out.append(p + off, static_cast(zero_run)); + } + off += static_cast(zero_run); + out.append(character_sets::hex + uint8_t(p[off]) * 4, 3); + ++off; + bits >>= static_cast(zero_run + 1); + } + if (off < width) { + out.append(p + off, width - off); + } +} +#endif // ADA_UNICODE_SSSE3 || ADA_NEON + +ada_really_inline void percent_encode_to_scalar(const char* p, const char* end, + const uint8_t character_set[], + std::string& out) { + for (; p != end; ++p) { + if (character_sets::bit_at(character_set, *p)) { + out.append(character_sets::hex + uint8_t(*p) * 4, 3); + } else { + out += *p; + } + } +} + +#if ADA_UNICODE_SSSE3 +// Classify 16 input bytes against the 32-byte character_set bitmap: +// bit_at(cs, b) == cs[b >> 3] & (1 << (b & 7)). pshufb looks up the two +// 16-byte halves of that bitmap; no per-call nibble table is built. +struct ssse3_percent_tables { + __m128i cs_lo; + __m128i cs_hi; + __m128i pow2; + __m128i mask_0f; + __m128i mask_07; + __m128i zero; +}; + +ADA_UNICODE_SIMD ssse3_percent_tables +load_ssse3_percent_tables(const uint8_t character_set[]) noexcept { + ssse3_percent_tables t{}; + t.cs_lo = _mm_loadu_si128(reinterpret_cast(character_set)); + t.cs_hi = + _mm_loadu_si128(reinterpret_cast(character_set + 16)); + // 1 << (0..7), duplicated so pshufb(index & 7) works for every lane. + t.pow2 = + _mm_setr_epi8(1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16, 32, 64, -128); + t.mask_0f = _mm_set1_epi8(0x0F); + t.mask_07 = _mm_set1_epi8(0x07); + t.zero = _mm_setzero_si128(); + return t; +} + +ADA_UNICODE_SIMD int ssse3_percent_mask( + __m128i word, const ssse3_percent_tables& tables) noexcept { + const __m128i idx = _mm_and_si128(_mm_srli_epi16(word, 3), tables.mask_0f); + const __m128i lo = _mm_shuffle_epi8(tables.cs_lo, idx); + const __m128i hi = _mm_shuffle_epi8(tables.cs_hi, idx); + // Bytes 0x80-0xFF are signed-negative; select the high half of the bitmap. + const __m128i high_byte = _mm_cmpgt_epi8(tables.zero, word); + const __m128i cs_byte = _mm_or_si128(_mm_and_si128(hi, high_byte), + _mm_andnot_si128(high_byte, lo)); + const __m128i bits = + _mm_shuffle_epi8(tables.pow2, _mm_and_si128(word, tables.mask_07)); + const __m128i hits = _mm_and_si128(cs_byte, bits); + return _mm_movemask_epi8(_mm_cmpeq_epi8(hits, tables.zero)) ^ 0xFFFF; +} + +ADA_UNICODE_SIMD void percent_encode_to_ssse3( + const char* p, const char* end, const uint8_t character_set[], + const ssse3_percent_tables& tables, std::string& out) { + // Pair 16-byte windows so a fully clean 32-byte run is one append. + while (p + 32 <= end) { + const __m128i word0 = _mm_loadu_si128(reinterpret_cast(p)); + const __m128i word1 = + _mm_loadu_si128(reinterpret_cast(p + 16)); + const int mask0 = ssse3_percent_mask(word0, tables); + const int mask1 = ssse3_percent_mask(word1, tables); + if ((mask0 | mask1) == 0) { + out.append(p, 32); + } else { + if (mask0 == 0) { + out.append(p, 16); + } else { + encode_mask_window(p, static_cast(mask0), 16, out); + } + if (mask1 == 0) { + out.append(p + 16, 16); + } else { + encode_mask_window(p + 16, static_cast(mask1), 16, out); + } + } + p += 32; + } + if (p + 16 <= end) { + const __m128i word = _mm_loadu_si128(reinterpret_cast(p)); + const int mask = ssse3_percent_mask(word, tables); + if (mask == 0) { + out.append(p, 16); + } else { + encode_mask_window(p, static_cast(mask), 16, out); + } + p += 16; + } + percent_encode_to_scalar(p, end, character_set, out); +} +#endif // ADA_UNICODE_SSSE3 + +#if ADA_NEON +ada_really_inline uint8x16x2_t +load_neon_percent_table(const uint8_t character_set[]) noexcept { + uint8x16x2_t table{}; + table.val[0] = vld1q_u8(character_set); + table.val[1] = vld1q_u8(character_set + 16); + return table; +} + +ada_really_inline uint8x16_t neon_percent_hits(uint8x16_t word, + uint8x16x2_t table) noexcept { + const uint8x16_t cs_bytes = vqtbl2q_u8(table, vshrq_n_u8(word, 3)); + const uint8x16_t bit_mask = vshlq_u8( + vdupq_n_u8(1), vreinterpretq_s8_u8(vandq_u8(word, vdupq_n_u8(7)))); + return vandq_u8(cs_bytes, bit_mask); +} + +ada_really_inline uint32_t neon_percent_mask(uint8x16_t hits) noexcept { + const uint8x16_t cmp = vcgtq_u8(hits, vdupq_n_u8(0)); + const uint8x16_t bit = {1, 2, 4, 8, 16, 32, 64, 128, + 1, 2, 4, 8, 16, 32, 64, 128}; + const uint8x16_t masked = vandq_u8(cmp, bit); + return static_cast(vaddv_u8(vget_low_u8(masked))) | + (static_cast(vaddv_u8(vget_high_u8(masked))) << 8); +} + +ada_really_inline void percent_encode_to_neon(const char* p, const char* end, + const uint8_t character_set[], + uint8x16x2_t table, + std::string& out) { + while (p + 32 <= end) { + const uint8x16_t hits0 = + neon_percent_hits(vld1q_u8(reinterpret_cast(p)), table); + const uint8x16_t hits1 = neon_percent_hits( + vld1q_u8(reinterpret_cast(p + 16)), table); + const bool clean0 = vmaxvq_u32(vreinterpretq_u32_u8(hits0)) == 0; + const bool clean1 = vmaxvq_u32(vreinterpretq_u32_u8(hits1)) == 0; + if (clean0 && clean1) { + out.append(p, 32); + } else { + if (clean0) { + out.append(p, 16); + } else { + encode_mask_window(p, neon_percent_mask(hits0), 16, out); + } + if (clean1) { + out.append(p + 16, 16); + } else { + encode_mask_window(p + 16, neon_percent_mask(hits1), 16, out); + } + } + p += 32; + } + if (p + 16 <= end) { + const uint8x16_t hits = + neon_percent_hits(vld1q_u8(reinterpret_cast(p)), table); + if (vmaxvq_u32(vreinterpretq_u32_u8(hits)) == 0) { + out.append(p, 16); + } else { + encode_mask_window(p, neon_percent_mask(hits), 16, out); + } + p += 16; + } + percent_encode_to_scalar(p, end, character_set, out); +} +#endif // ADA_NEON + +#if ADA_RVV +ada_really_inline void percent_encode_to_rvv(const char* p, const char* end, + const uint8_t character_set[], + std::string& out) { + while (p < end) { + const size_t remaining = static_cast(end - p); + const size_t vl = __riscv_vsetvl_e8m1(remaining); + const vuint8m1_t word = + __riscv_vle8_v_u8m1(reinterpret_cast(p), vl); + const vuint8m1_t cs_bytes = + __riscv_vluxei8(character_set, __riscv_vsrl(word, 3, vl), vl); + const vuint8m1_t bit_mask = __riscv_vsll(__riscv_vmv_v_x_u8m1(1, vl), + __riscv_vand(word, 7, vl), vl); + const long idx = __riscv_vfirst( + __riscv_vmsne(__riscv_vand(cs_bytes, bit_mask, vl), 0, vl), vl); + if (idx < 0) { + out.append(p, vl); + p += vl; + continue; + } + if (idx > 0) { + out.append(p, static_cast(idx)); + p += idx; + } + out.append(character_sets::hex + uint8_t(*p) * 4, 3); + ++p; + } +} +#endif // ADA_RVV + +#if ADA_UNICODE_SSSE3 || ADA_NEON || ADA_RVV +// Setter and existing percent_encode benches are 2-44 bytes. Table setup +// plus mask walking costs more instructions than bit_at on those inputs +// (especially dense USERINFO). SIMD pays off on the remaining suffix. +static constexpr size_t kPercentEncodeSimdMin = 48; + +void percent_encode_to_wide(const char* p, const char* end, + const uint8_t character_set[], std::string& out) { + // Worst case every byte becomes %XX. Avoids realloc while walking windows. + out.reserve(out.size() + static_cast(end - p) * 3); +#if ADA_UNICODE_SSSE3 + const ssse3_percent_tables tables = load_ssse3_percent_tables(character_set); + percent_encode_to_ssse3(p, end, character_set, tables, out); +#elif ADA_NEON + percent_encode_to_neon(p, end, character_set, + load_neon_percent_table(character_set), out); +#elif ADA_RVV + percent_encode_to_rvv(p, end, character_set, out); +#endif +} +#endif + +} // namespace + +void percent_encode_suffix(const char* p, const char* end, + const uint8_t character_set[], std::string& out) { +#if ADA_UNICODE_SSSE3 || ADA_NEON || ADA_RVV + if (static_cast(end - p) >= kPercentEncodeSimdMin) { + percent_encode_to_wide(p, end, character_set, out); + return; + } +#endif + percent_encode_to_scalar(p, end, character_set, out); +} + +} // namespace ada::unicode diff --git a/tests/basic_tests.cpp b/tests/basic_tests.cpp index 8be119294..b0aa02019 100644 --- a/tests/basic_tests.cpp +++ b/tests/basic_tests.cpp @@ -713,6 +713,139 @@ TEST(basic_tests, percent_decode_direct) { ASSERT_EQ(percent_decode("%%41", 0), "%A"); // '%' then a valid escape } +namespace { +size_t scalar_percent_encode_index(std::string_view input, + const uint8_t character_set[]) { + for (size_t i = 0; i < input.size(); i++) { + if (ada::character_sets::bit_at(character_set, input[i])) { + return i; + } + } + return input.size(); +} + +std::string scalar_percent_encode(std::string_view input, + const uint8_t character_set[]) { + std::string out; + for (unsigned char c : input) { + if (ada::character_sets::bit_at(character_set, c)) { + out.append(ada::character_sets::hex + static_cast(c) * 4, 3); + } else { + out.push_back(static_cast(c)); + } + } + return out; +} + +const uint8_t* percent_encode_sets[] = { + ada::character_sets::C0_CONTROL_PERCENT_ENCODE, + ada::character_sets::FRAGMENT_PERCENT_ENCODE, + ada::character_sets::QUERY_PERCENT_ENCODE, + ada::character_sets::SPECIAL_QUERY_PERCENT_ENCODE, + ada::character_sets::USERINFO_PERCENT_ENCODE, + ada::character_sets::PATH_PERCENT_ENCODE, + ada::character_sets::WWW_FORM_URLENCODED_PERCENT_ENCODE, +}; +} // namespace + +TEST(basic_tests, percent_encode_index_and_encode_match_scalar) { + for (const uint8_t* character_set : percent_encode_sets) { + for (size_t len = 0; len <= 80; len++) { + std::string clean(len, 'a'); + ASSERT_EQ(ada::unicode::percent_encode_index(clean, character_set), + clean.size()) + << "clean len=" << len; + ASSERT_EQ(ada::unicode::percent_encode(clean, character_set), clean) + << "clean len=" << len; + + for (size_t pos = 0; pos < len; pos++) { + std::string one_space = clean; + one_space[pos] = ' '; + ASSERT_EQ(ada::unicode::percent_encode_index(one_space, character_set), + scalar_percent_encode_index(one_space, character_set)) + << "space at " << pos << " len=" << len; + ASSERT_EQ(ada::unicode::percent_encode(one_space, character_set), + scalar_percent_encode(one_space, character_set)) + << "space at " << pos << " len=" << len; + + std::string one_high = clean; + one_high[pos] = static_cast(0xE1); + ASSERT_EQ(ada::unicode::percent_encode_index(one_high, character_set), + pos) + << "high at " << pos << " len=" << len; + ASSERT_EQ(ada::unicode::percent_encode(one_high, character_set), + scalar_percent_encode(one_high, character_set)) + << "high at " << pos << " len=" << len; + } + } + + std::string dense(32, '"'); + ASSERT_EQ(ada::unicode::percent_encode_index(dense, character_set), + scalar_percent_encode_index(dense, character_set)); + ASSERT_EQ(ada::unicode::percent_encode(dense, character_set), + scalar_percent_encode(dense, character_set)); + + std::string long_dense(64, '"'); + ASSERT_EQ(ada::unicode::percent_encode(long_dense, character_set), + scalar_percent_encode(long_dense, character_set)); + + for (size_t len : {96u, 128u, 256u}) { + std::string clean(len, 'a'); + ASSERT_EQ(ada::unicode::percent_encode(clean, character_set), clean) + << "clean len=" << len; + clean[len / 2] = ' '; + clean[len - 1] = static_cast(0x7F); + ASSERT_EQ(ada::unicode::percent_encode(clean, character_set), + scalar_percent_encode(clean, character_set)) + << "wide len=" << len; + } + + std::string mixed = std::string(15, 'a') + "|" + std::string(16, 'b') + + std::string(1, char(0x7F)) + std::string(17, 'c'); + ASSERT_EQ(ada::unicode::percent_encode_index(mixed, character_set), + scalar_percent_encode_index(mixed, character_set)); + ASSERT_EQ(ada::unicode::percent_encode(mixed, character_set), + scalar_percent_encode(mixed, character_set)); + const size_t idx = ada::unicode::percent_encode_index(mixed, character_set); + ASSERT_EQ(ada::unicode::percent_encode(mixed, character_set, idx), + ada::unicode::percent_encode(mixed, character_set)); + } +} + +TEST(basic_tests, percent_encode_template_append_and_replace) { + const uint8_t* query = ada::character_sets::QUERY_PERCENT_ENCODE; + const std::string clean(24, 'n'); + const std::string dirty = std::string(16, 'n') + " " + std::string(16, 'n'); + const std::string long_dirty = + std::string(16, 'n') + " " + std::string(48, 'n'); + + std::string replace_clean; + ASSERT_FALSE( + ada::unicode::percent_encode(clean, query, replace_clean)); + ASSERT_TRUE(replace_clean.empty()); + + std::string replace_dirty; + ASSERT_TRUE(ada::unicode::percent_encode(dirty, query, replace_dirty)); + ASSERT_EQ(replace_dirty, scalar_percent_encode(dirty, query)); + + std::string replace_long; + ASSERT_TRUE( + ada::unicode::percent_encode(long_dirty, query, replace_long)); + ASSERT_EQ(replace_long, scalar_percent_encode(long_dirty, query)); + + std::string append_out = "pre:"; + ASSERT_FALSE(ada::unicode::percent_encode(clean, query, append_out)); + ASSERT_EQ(append_out, "pre:"); + + ASSERT_TRUE(ada::unicode::percent_encode(dirty, query, append_out)); + ASSERT_EQ(append_out, "pre:" + scalar_percent_encode(dirty, query)); + + ASSERT_TRUE( + ada::unicode::percent_encode(long_dirty, query, append_out)); + ASSERT_EQ(append_out, "pre:" + scalar_percent_encode(dirty, query) + + scalar_percent_encode(long_dirty, query)); +} + // Regression: try_can_parse_absolute_fast returned true for a valid IPv4 host // without validating the port. For "wS://1.3.3.51.:+" the host "1.3.3.51." // passes the IPv4 fast path, but the port "+" is not a valid digit, so the