Windows support 1/x: centipede:util

This mainly add windows support for the on-demand mmap and make sure that file I/Os are in the binary mode to avoid the automatic CRLF conversion, which is undesired at least in the tests.

Also add a Bazel platform for building under Windows and a GitHub action for testing under Windows.

PiperOrigin-RevId: 959258384

Author: xinhaoyuan

.github/workflows/bazel_test_centipede.yml

@@ -30,6 +30,8 @@ jobs:
     # TODO(xinhaoyuan): Bump to 24.04 after https://github.com/llvm/llvm-project/issues/102443
     # is fixed.
     runs-on: ubuntu-22.04
+    permissions:
+      contents: read
     timeout-minutes: 60
     strategy:
       matrix:
@@ -39,13 +41,13 @@ jobs:
         run: |
           sudo sysctl -w kernel.core_pattern=""
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
       - name: Install dependencies
         run: |
           sudo apt-get update && sudo apt-get install -yq \
             clang llvm libssl-dev
       - name: Restore latest cache
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25  # v5.1.0
         with:
           path: "~/.cache/bazel"
           key: bazel-centipede-cache-${{ matrix.config }}
@@ -84,19 +86,21 @@ jobs:
           bazel test --no//fuzztest:use_riegeli --test_output=errors --linkopt=-fsanitize=address --copt=-fsanitize=address --test_env=ASAN_OPTIONS=detect_leaks=0 --platform_suffix=asan --test_timeout=600 centipede/puzzles:all
       - name: Save new cache based on main
         if: github.ref == 'refs/heads/main'
-        uses: actions/cache/save@v4
+        uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25  # v5.1.0
         with:
           path: "~/.cache/bazel"
           key: bazel-centipede-cache-${{ matrix.config }}-${{ github.run_id }}
   run_tests_mac:
     name: Run Centipede tests (MacOS)
     runs-on: macos-15
+    permissions:
+      contents: read
     timeout-minutes: 60
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
       - name: Restore latest cache
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25  # v5.1.0
         with:
           path: "~/.cache/bazel"
           key: bazel-centipede-cache-mac
@@ -127,7 +131,37 @@ jobs:
           bazel --output_user_root="${HOME}/.cache/bazel" test --test_output=errors --no//fuzztest:use_riegeli --linkopt=-fsanitize=address --copt=-fsanitize=address --test_env=ASAN_OPTIONS=detect_leaks=0 --platform_suffix=asan --test_timeout=600 centipede/puzzles:all
       - name: Save new cache based on main
         if: github.ref == 'refs/heads/main'
-        uses: actions/cache/save@v4
+        uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25  # v5.1.0
         with:
           path: "~/.cache/bazel"
           key: bazel-centipede-cache-mac-${{ github.run_id }}
+  run_tests_win:
+    name: Run Centipede tests (Windows)
+    runs-on: windows-latest
+    permissions:
+      contents: read
+    timeout-minutes: 60
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+      - name: Restore latest cache
+        uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25  # v5.1.0
+        with:
+          path: "~/.cache/bazel"
+          key: bazel-centipede-cache-win-${{ matrix.config }}
+          restore-keys: bazel-centipede-cache-win-${{ matrix.config }}-
+      - name: Set environment variable
+        run: echo "USE_BAZEL_VERSION=8.7.0" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+      - name: Run unit tests
+        if: ${{ !cancelled() }}
+        run: |
+          <# Only supported libraries are tested here. #> `
+          bazelisk test --disk_cache=~/.cache/bazel --local_test_jobs=1 --test_output=errors --no//fuzztest:use_riegeli `
+          --extra_toolchains=@local_config_cc//:cc-toolchain-x64_windows-clang-cl `
+          --extra_execution_platforms=//:x64_windows-clang-cl -- centipede:util_test
+      - name: Save new cache based on main
+        # if: github.ref == 'refs/heads/main'
+        uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25  # v5.1.0
+        with:
+          path: "~/.cache/bazel"
+          key: bazel-centipede-cache-win-${{ github.run_id }}

BUILD

@@ -13,3 +13,12 @@
 # limitations under the License.
 
 exports_files(["MODULE.bazel"])
+
+platform(
+    name = "x64_windows-clang-cl",
+    constraint_values = [
+        "@platforms//cpu:x86_64",
+        "@platforms//os:windows",
+        "@bazel_tools//tools/cpp:clang-cl",
+    ],
+)

MODULE.bazel

@@ -23,6 +23,10 @@ bazel_dep(
     name = "rules_cc",
     version = "0.2.17",
 )
+
+cc_configure = use_extension("@rules_cc//cc:extensions.bzl", "cc_configure_extension")
+use_repo(cc_configure, "local_config_cc")
+
 bazel_dep(
     name = "rules_shell",
     version = "0.6.1",

centipede/BUILD

@@ -211,12 +211,18 @@ cc_library(
 )
 
 # Various utilities.
+cc_library(
+    name = "windows_includes",
+    hdrs = ["windows_includes.h"],
+)
+
 cc_library(
     name = "util",
     srcs = ["util.cc"],
     hdrs = ["util.h"],
     deps = [
         ":feature",
+        ":windows_includes",
         "@abseil-cpp//absl/base:core_headers",
         "@abseil-cpp//absl/base:nullability",
         "@abseil-cpp//absl/strings",
@@ -1385,7 +1391,12 @@ cc_test(
 cc_test(
     name = "util_test",
     srcs = ["util_test.cc"],
-    copts = ["-fno-signed-char"],
+    copts = select({
+        "@platforms//os:windows": [
+            "/J",  # Make unsigned char the default
+        ],
+        "//conditions:default": ["-fno-signed-char"],
+    }),
     deps = [
         ":feature",
         ":thread_pool",

centipede/util.cc

@@ -17,8 +17,12 @@
 
 #include "./centipede/util.h"
 
+#if defined(_WIN32)
+#include "./centipede/windows_includes.h"
+#else
 #include <sys/mman.h>
 #include <unistd.h>
+#endif
 
 #include <algorithm>
 #include <cctype>
@@ -47,6 +51,7 @@
 #include "absl/base/const_init.h"
 #include "absl/base/nullability.h"
 #include "absl/base/thread_annotations.h"
+#include "absl/strings/match.h"
 #include "absl/strings/str_format.h"
 #include "absl/strings/str_replace.h"
 #include "absl/strings/str_split.h"
@@ -63,8 +68,13 @@ namespace fuzztest::internal {
 
 size_t GetRandomSeed(size_t seed) {
   if (seed != 0) return seed;
+#if defined(_WIN32)
+  return time(nullptr) + GetCurrentProcessId() +
+         std::hash<std::thread::id>{}(std::this_thread::get_id());
+#else
   return time(nullptr) + getpid() +
          std::hash<std::thread::id>{}(std::this_thread::get_id());
+#endif
 }
 
 std::string AsPrintableString(ByteSpan data, size_t max_len) {
@@ -83,7 +93,7 @@ std::string AsPrintableString(ByteSpan data, size_t max_len) {
 
 template <typename Container>
 void ReadFromLocalFile(std::string_view file_path, Container &data) {
-  std::ifstream f(std::string{file_path});
+  std::ifstream f(std::string{file_path}, std::ios::in | std::ios::binary);
   if (!f) return;
   f.seekg(0, std::ios_base::end);
   auto size = f.tellg();
@@ -112,12 +122,13 @@ void ReadFromLocalFile(std::string_view file_path,
 }
 
 void ClearLocalFileContents(std::string_view file_path) {
-  std::ofstream f(std::string{file_path}, std::ios::out | std::ios::trunc);
+  std::ofstream f(std::string{file_path},
+                  std::ios::out | std::ios::trunc | std::ios::binary);
   FUZZTEST_CHECK(f) << "Failed to clear the file: " << file_path;
 }
 
 void WriteToLocalFile(std::string_view file_path, ByteSpan data) {
-  std::ofstream f(std::string{file_path});
+  std::ofstream f(std::string{file_path}, std::ios::out | std::ios::binary);
   FUZZTEST_CHECK(f) << "Failed to open local file: " << file_path;
   f.write(reinterpret_cast<const char *>(data.data()),
           static_cast<int64_t>(data.size()));
@@ -136,13 +147,15 @@ void WriteToLocalFile(std::string_view file_path, const FeatureVec &data) {
 
 void WriteToLocalHashedFileInDir(std::string_view dir_path, ByteSpan data) {
   if (dir_path.empty()) return;
-  std::string file_path = std::filesystem::path(dir_path).append(Hash(data));
+  std::string file_path =
+      std::filesystem::path(dir_path).append(Hash(data)).string();
   WriteToLocalFile(file_path, data);
 }
 
 void WriteToRemoteHashedFileInDir(std::string_view dir_path, ByteSpan data) {
   if (dir_path.empty()) return;
-  std::string file_path = std::filesystem::path(dir_path).append(Hash(data));
+  std::string file_path =
+      std::filesystem::path(dir_path).append(Hash(data)).string();
   FUZZTEST_CHECK_OK(
       RemoteFileSetContents(file_path, std::string(data.begin(), data.end())));
 }
@@ -155,17 +168,24 @@ std::string HashOfFileContents(std::string_view file_path) {
 }
 
 std::string ProcessAndThreadUniqueID(std::string_view prefix) {
-  // operator << is the only way to serialize std::this_thread::get_id().
   std::ostringstream oss;
+#if defined(_WIN32)
+  oss << prefix << GetCurrentProcessId() << "-" << GetCurrentThreadId();
+#else
+  // operator << is the only way to serialize std::this_thread::get_id().
   oss << prefix << getpid() << "-" << std::this_thread::get_id();
+#endif
   return oss.str();
 }
 
 std::string TemporaryLocalDirPath() {
   const char *TMPDIR = getenv("TMPDIR");
+  if (!TMPDIR) TMPDIR = getenv("TEMP");
+  if (!TMPDIR) TMPDIR = getenv("TMP");
   std::string tmp = TMPDIR ? TMPDIR : "/tmp";
-  return std::filesystem::path(tmp).append(
-      ProcessAndThreadUniqueID("centipede-"));
+  return std::filesystem::path(tmp)
+      .append(ProcessAndThreadUniqueID("centipede-"))
+      .string();
 }
 
 // We need to maintain a global set of dirs that CreateLocalDirRemovedAtExit()
@@ -189,13 +209,15 @@ static void RemoveDirsAtExit() {
 
 void CreateLocalDirRemovedAtExit(std::string_view path) {
   // Safeguard against removing dirs not created by TemporaryLocalDirPath().
-  FUZZTEST_CHECK_NE(path.find("/centipede-"), std::string::npos);
+  FUZZTEST_CHECK(absl::StrContains(path, "/centipede-") ||
+                 absl::StrContains(path, "\\centipede-"));
   // Create the dir.
   std::error_code error;
-  std::filesystem::remove_all(path, error);
-  FUZZTEST_LOG_IF(ERROR, error)
-      << "Unable to clean up existing dir " << path << ": " << error.message();
-  std::filesystem::create_directories(path);
+  std::filesystem::path p(path);
+  if (std::filesystem::exists(p, error)) {
+    std::filesystem::remove_all(p, error);
+  }
+  std::filesystem::create_directories(p, error);
   // Add to dirs_to_delete_at_exit.
   absl::MutexLock lock(dirs_to_delete_at_exit_mutex);
   if (!dirs_to_delete_at_exit) {
@@ -206,7 +228,7 @@ void CreateLocalDirRemovedAtExit(std::string_view path) {
 }
 
 ScopedFile::ScopedFile(std::string_view dir_path, std::string_view name)
-    : my_path_(std::filesystem::path(dir_path) / name) {}
+    : my_path_((std::filesystem::path(dir_path) / name).string()) {}
 
 ScopedFile::~ScopedFile() {
   std::error_code error;
@@ -359,16 +381,53 @@ std::vector<size_t> RandomWeightedSubset(absl::Span<const uint64_t> set,
   return res;
 }
 
+#if defined(_WIN32)
+static LONG CALLBACK
+AutoCommitPageFaultHandler(PEXCEPTION_POINTERS ExceptionInfo) {
+  if (ExceptionInfo->ExceptionRecord->ExceptionCode ==
+      EXCEPTION_ACCESS_VIOLATION) {
+    ULONG_PTR fault_addr =
+        ExceptionInfo->ExceptionRecord->ExceptionInformation[1];
+    if (VirtualAlloc(reinterpret_cast<void*>(fault_addr), 1, MEM_COMMIT,
+                     PAGE_READWRITE) != nullptr) {
+      return EXCEPTION_CONTINUE_EXECUTION;
+    }
+  }
+  return EXCEPTION_CONTINUE_SEARCH;
+}
+#endif
+
 uint8_t *MmapNoReserve(size_t size) {
+#if defined(_WIN32)
+  // Set up page fault handler to commit page on demand.
+  static bool installed_veh = []() {
+    // Must use `First=0` as it could otherwise conflict with e.g. sanitizers.
+    AddVectoredExceptionHandler(/*First=*/0, AutoCommitPageFaultHandler);
+    return true;
+  }();
+  (void)installed_veh;
+  // MEM_RESERVE has different semantics and does not contradict with
+  // MAP_NORESERVE for mmap.
+  auto result = VirtualAlloc(nullptr, size, MEM_RESERVE, PAGE_READWRITE);
+  FUZZTEST_CHECK(result != nullptr)
+      << "VirtualAlloc failed for size " << size << " err=" << GetLastError();
+  return reinterpret_cast<uint8_t*>(result);
+#else
   auto result = mmap(0, size, PROT_READ | PROT_WRITE,
                      MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0);
   FUZZTEST_CHECK(result != MAP_FAILED);
   return reinterpret_cast<uint8_t *>(result);
+#endif
 }
 
 void Munmap(uint8_t *ptr, size_t size) {
+#if defined(_WIN32)
+  BOOL result = VirtualFree(ptr, 0, MEM_RELEASE);
+  FUZZTEST_CHECK(result != 0);
+#else
   auto result = munmap(ptr, size);
   FUZZTEST_CHECK_EQ(result, 0);
+#endif
 }
 
 int PollTimeoutMs(absl::Duration timeout) {

centipede/util_test.cc

@@ -32,6 +32,10 @@
 #include "./common/hash.h"
 #include "./common/logging.h"
 
+#if defined(_WIN32)
+#define setenv(n, v, _r) _putenv_s(n, v)
+#endif
+
 namespace fuzztest::internal {
 
 TEST(UtilTest, AsString) {
@@ -116,7 +120,8 @@ TEST(UtilTest, TemporaryLocalDirPath) {
     auto temp_dir = TemporaryLocalDirPath();
     // Create dir, create a file there, write to file, read from it, remove dir.
     std::filesystem::create_directories(temp_dir);
-    std::string temp_file_path = std::filesystem::path(temp_dir).append("blah");
+    std::string temp_file_path =
+        std::filesystem::path(temp_dir).append("blah").string();
     ByteArray written_data{1, 2, 3};
     WriteToLocalFile(temp_file_path, written_data);
     ByteArray read_data;
@@ -162,8 +167,8 @@ TEST(UtilTest, CreateLocalDirRemovedAtExit) {
   EXPECT_TRUE(std::filesystem::exists(tmpdir));
   setenv("CENTIPEDE_UTIL_TEST_TEMP_DIR", tmpdir.c_str(), 1);
   // Create two subdirs via CreateLocalDirRemovedAtExit.
-  std::string subdir1 = std::filesystem::path(tmpdir).append("1");
-  std::string subdir2 = std::filesystem::path(tmpdir).append("2");
+  std::string subdir1 = std::filesystem::path(tmpdir).append("1").string();
+  std::string subdir2 = std::filesystem::path(tmpdir).append("2").string();
   CreateLocalDirRemovedAtExit(subdir1);
   CreateLocalDirRemovedAtExit(subdir2);
   EXPECT_TRUE(std::filesystem::exists(subdir1));
@@ -293,6 +298,15 @@ TEST(UtilTest, RemoveSubset) {
               testing::ElementsAre(std::vector<int>{1}, std::vector<int>{3}));
 }
 
+TEST(UtilTest, MmapTest) {
+  static constexpr size_t kBufSize = 1 << 30;  // 1 GiB
+  auto* buf = MmapNoReserve(kBufSize);
+  ASSERT_NE(buf, nullptr);
+  EXPECT_EQ(buf[1234], 0);
+  EXPECT_EQ(buf[567890], 0);
+  Munmap(buf, kBufSize);
+}
+
 TEST(UtilTest, PollTimeoutMsWorks) {
   EXPECT_GT(PollTimeoutMs(absl::ZeroDuration()), 0);
   EXPECT_GT(PollTimeoutMs(-absl::InfiniteDuration()), 0);