From ce6bc0a6dfe14392e57c03aecc1de7be9b1c222c Mon Sep 17 00:00:00 2001 From: Christoforos Date: Sat, 1 Aug 2026 21:45:21 +0300 Subject: [PATCH 1/5] Added volesti benchmark suite module --- .gitignore | 6 ++++ CMakeLists.txt | 29 ++++++++++++++++- README.md | 2 +- benchmark/available_choices.md | 54 +++++++++++++++++++++++++++++++ benchmark/plot.py | 53 ++++++++++++++++++++++++++++++ benchmark/run_benchmark.py | 40 +++++++++++++++++++++++ benchmark/walk_config.json | 35 ++++++++++++++++++++ external/volesti | 2 +- pyproject.toml | 7 ++++ src/bindings/volesti_bindings.cpp | 31 ++++++++++++++++++ volestipy/__init__.py | 2 ++ 11 files changed, 258 insertions(+), 3 deletions(-) create mode 100644 benchmark/available_choices.md create mode 100644 benchmark/plot.py create mode 100644 benchmark/run_benchmark.py create mode 100644 benchmark/walk_config.json diff --git a/.gitignore b/.gitignore index 1c5f0ac..d82883a 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,9 @@ Thumbs.db htmlcov/ .tox/ external/lp_solve_5.5/ +Volesti-env/ +test.py +*.tar.gz +benchmark/__pycache__/ +benchmark/*.csv +benchmark/results/ \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index fa68856..ac60dbf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,6 +34,15 @@ if(NOT pybind11_FOUND) find_package(pybind11 REQUIRED HINTS "${pybind11_DIR}") endif() +# --- QD library --- +find_library(QD_LIB qd) + +if(NOT QD_LIB) + message(FATAL_ERROR "Could not find QD library") +endif() + +message(STATUS "QD library: ${QD_LIB}") + # --- Find Eigen3 --- find_package(Eigen3 3.3 REQUIRED NO_MODULE) if(NOT Eigen3_FOUND) @@ -52,7 +61,7 @@ endif() # --- Find Boost --- # volesti only needs Boost headers (random, math) - no compiled libraries required -find_package(Boost 1.56 REQUIRED) +find_package(Boost 1.56 REQUIRED COMPONENTS program_options) if(NOT Boost_FOUND) # Fallback: locate headers manually find_path(Boost_INCLUDE_DIRS NAMES boost/random.hpp @@ -150,23 +159,41 @@ endif() # --- Build the extension --- pybind11_add_module(_volestipy MODULE src/bindings/volesti_bindings.cpp + + ${CMAKE_CURRENT_SOURCE_DIR}/external/volesti/benchmark/src/benchmark_cli.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/external/volesti/benchmark/src/benchmark_utils.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/external/volesti/benchmark/src/walk_parameters.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/external/volesti/benchmark/src/walk_registry.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/external/volesti/benchmark/src/walk_result.cpp + ${LPSOLVE_SOURCES} ) target_include_directories(_volestipy PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include # project local headers (sampling_minimal.hpp) ${VOLESTI_INCLUDE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/external/volesti ${VOLESTI_INCLUDE_DIR}/../external # Spectra, minimum_ellipsoid, etc. + ${VOLESTI_INCLUDE_DIR}/generators + ${VOLESTI_INCLUDE_DIR}/include/generators + ${VOLESTI_INCLUDE_DIR}/include/preprocess + ${VOLESTI_INCLUDE_DIR}/preprocess + ${VOLESTI_INCLUDE_DIR}/benchmark/include + ${CMAKE_CURRENT_SOURCE_DIR}/external/volesti/benchmark/include ${Boost_INCLUDE_DIRS} ${LPSOLVE_INCLUDE_DIRS} ) target_link_libraries(_volestipy PRIVATE Eigen3::Eigen + Boost::program_options + ${QD_LIB} + m ) target_compile_definitions(_volestipy PRIVATE ${LPSOLVE_DEFINITIONS} + DISABLE_NLP_ORACLES ) target_compile_options(_volestipy PRIVATE diff --git a/README.md b/README.md index 9c2766b..81e78fb 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ sudo apt-get install -y \ ### 3. Install Python dependencies ```bash -pip install pybind11 numpy +pip install pybind11 "numpy<2.0" ``` ### 4a. Build via `pip` (recommended) diff --git a/benchmark/available_choices.md b/benchmark/available_choices.md new file mode 100644 index 0000000..0cc32ed --- /dev/null +++ b/benchmark/available_choices.md @@ -0,0 +1,54 @@ +# Available Choices + +## Target ESS + +The code will use a dynamic batch size in order to sample the necessary number of samples needed to reach the targert ESS + +## Time limit + +The maximum time the code runs. If the time runs out, the code will stop at the first available opportunity. + +## Base seed + +The seed used by the random number generator. Use any number you like. + +## Dimension + +The number of dimesnion the generated polytope will have. This only applies to polytopes generated by the generator functions. It is ignored for custom polytopes. + +## Rotation angle + +You can rotate the input polytope by this number over all pairs of dimensions. Angle is in degrees. Use 0 to avoid rotation. + +## Polytope Choice + +Choose the polytope that will be sampled from. +Available polytope choices are: Cube, Simplex, Birkhoff, Cross, OrderPolytope, Custom + +## Custom A file and Custom b file + +These are the names of the A and b files of your custom polytope defining the polytope Ax<=b. Csv files are expected, with no labels. +Place them inside build folder next to the executable. + +## Dynamic batch size + +If you have this choice to true, the code will run a dynamic batch size in order to hit target ESS. +If you have this choice to false, the code will generate exactly "samples" amount of samples, the parameter defined for each walk method. + +## Write to file + +Set this to true in order to create txt files holding all the generated samples. Useful for later use of the samples. + +## Rounding + +If this is set to true, the code will firrst call a rounding function on the input polytope, the john position. +The sampling will take place on the rounded polytope before the sample are reverted back to the original and returned. + +## Walks + +For each walk, you can change these parameters: + +- enabled: This will determine if this method will run or not when sampling +- samples: How many samples the code will take if dynamic batch size is also false. If dynamic batch size is on, target ESS will be used instead to determine when the execution stops. +- walk_len: It is a thinning parameter. The code will save the sample only after walk_len samples are sampled. +walk_len = walk_base + dim * walk_multiplier" diff --git a/benchmark/plot.py b/benchmark/plot.py new file mode 100644 index 0000000..1fe7a1d --- /dev/null +++ b/benchmark/plot.py @@ -0,0 +1,53 @@ +import pandas as pd +import matplotlib.pyplot as plt +import os + +def plot_results(csv_file, start_row=0): + """Reads the benchmark CSV and plots data starting from start_row.""" + if not os.path.exists(csv_file): + print(f"Error: {csv_file} was not generated or found.") + return + + df = pd.read_csv(csv_file, skipinitialspace=True) + df.columns = df.columns.str.strip() + + if start_row > 0: + df = df.iloc[start_row:] + + if df.empty: + print("No new data to plot.") + return + + print(f"Plotting {len(df)} new rows of data...") + + fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 6)) + + for method_name, method_data in df.groupby('Method'): + method_data = method_data.sort_values(by='Dimension') + + ax1.plot(method_data['Dimension'], method_data['Time_Sec'], + marker='o', label=method_name.strip()) + + ax2.plot(method_data['Dimension'], method_data['Mixing_Ratio'], + marker='o', label=method_name.strip()) + + ax1.set_yscale('log') + ax1.set_title('Time vs. Dimension') + ax1.set_xlabel('Dimension') + ax1.set_ylabel('Total Algorithm Time (Seconds)') + ax1.grid(True, which="both", linestyle='--', alpha=0.7) + ax1.legend() + + ax2.set_yscale('log') + ax2.set_title('Mixing Ratio vs. Dimension') + ax2.set_xlabel('Dimension') + ax2.set_ylabel('Mixing Ratio (Steps / ESS)') + ax2.grid(True, which="both", linestyle='--', alpha=0.7) + ax2.legend() + + plt.tight_layout() + plt.show() + +if __name__ == "__main__": + # If run directly, plot everything (start_row=0) + plot_results("benchmark_results.csv") \ No newline at end of file diff --git a/benchmark/run_benchmark.py b/benchmark/run_benchmark.py new file mode 100644 index 0000000..58c6288 --- /dev/null +++ b/benchmark/run_benchmark.py @@ -0,0 +1,40 @@ +import volestipy +import os +import pandas as pd +from plot import plot_results + +def main(): + csv_file = "benchmark_results.csv" + + existing_rows = 0 + if os.path.exists(csv_file): + try: + existing_rows = len(pd.read_csv(csv_file)) + except pd.errors.EmptyDataError: + pass # File exists but is empty + + script_dir = os.path.dirname(os.path.abspath(__file__)) + config_path = os.path.abspath(os.path.join(script_dir, "walk_config.json")) + + print("\n--- Starting C++ Benchmark ---") + + try: + volestipy.run_benchmark([ + "--config", config_path + ]) + print("--- C++ Benchmark Finished Successfully ---\n") + + except KeyboardInterrupt: + print("\n\n!!! Benchmark Interrupted by User (Ctrl+C) !!!") + print("Salvaging data collected so far...\n") + + except Exception as e: + print(f"\n\n!!! Benchmark Crashed with Error: {e} !!!") + print("Salvaging data collected so far...\n") + + finally: + # Pass the row offset so we ONLY plot the new stuff! + plot_results(csv_file, start_row=existing_rows) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/benchmark/walk_config.json b/benchmark/walk_config.json new file mode 100644 index 0000000..fa3448c --- /dev/null +++ b/benchmark/walk_config.json @@ -0,0 +1,35 @@ +{ + "global_settings": { + "target_ESS": 3000, + "time_limit_sec": 30.0, + "base_seed": 42, + "dimensions": [10,20,30,40,50,60], + "rotation_angle": 53, + "polytope_choice": "Cube", + "custom_A_file": "Cube_50_A.csv", + "custom_b_file": "Cube_50_b.csv", + "dynamic_batch_size": true, + "write_to_file": true, + "rounding": false, + "auto_walk": false, + "show_menu": false + }, +"walks": { + "BallWalk": { "enabled": false, "samples": 20000, "walk_len_multiplier": 2, "walk_len_base": 0 }, + "BilliardWalk": { "enabled": true, "samples": 2000, "walk_len_multiplier": 0, "walk_len_base": 1 }, + "AcceleratedBilliardWalk": { "enabled": true, "samples": 2000, "walk_len_multiplier": 0, "walk_len_base": 1 }, + "SparseBilliardWalk": { "enabled": false, "samples": 1000, "walk_len_multiplier": 0, "walk_len_base": 1 }, + "CDHRWalk": { "enabled": true, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0 }, + "RDHRWalk": { "enabled": false, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0 }, + "DikinWalk": { "enabled": false, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0 }, + "JohnWalk": { "enabled": false, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0 }, + "VaidyaWalk": { "enabled": false, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0 }, + "GaussianBallWalk": { "enabled": false, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0, "a_i_param": 1.0 }, + "GaussianCDHRWalk": { "enabled": false, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0, "a_i_param": 1.0 }, + "BilliardShakeAndBakeWalk": { "enabled": false, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0 }, + "ShakeAndBakeWalk": { "enabled": false, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0 }, + "BCDHRWalk": { "enabled": false, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0 }, + "BRDHRWalk": { "enabled": false, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0 }, + "CRHMCWalk": { "enabled": false, "samples": 1000, "walk_len_multiplier": 0, "walk_len_base": 1 } + } +} \ No newline at end of file diff --git a/external/volesti b/external/volesti index 410a90f..6e8191f 160000 --- a/external/volesti +++ b/external/volesti @@ -1 +1 @@ -Subproject commit 410a90f7903edf189f2645424bdada69f28a0a69 +Subproject commit 6e8191f2ee52effaf24ed6d3af0cf5c0bd1b87c0 diff --git a/pyproject.toml b/pyproject.toml index 337130e..6ec90b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,3 +32,10 @@ dev = ["pytest>=7.0", "matplotlib", "scipy"] [project.urls] Repository = "https://github.com/GeomScale/volestipy" + +[tool.setuptools] +package-dir = {"" = "."} + +[tool.setuptools.packages.find] +where = ["."] +include = ["volestipy*"] diff --git a/src/bindings/volesti_bindings.cpp b/src/bindings/volesti_bindings.cpp index 74f5346..49fd49d 100644 --- a/src/bindings/volesti_bindings.cpp +++ b/src/bindings/volesti_bindings.cpp @@ -54,6 +54,9 @@ #include "preprocess/min_sampling_covering_ellipsoid_rounding.hpp" #include "preprocess/inscribed_ellipsoid_rounding.hpp" +// For Benchmark +#include "benchmark/include/benchmark_cli.hpp" + #include #include #include @@ -998,4 +1001,32 @@ R : float // Version info m.attr("__version__") = "0.1.0"; m.attr("__volesti_version__") = "1.1.2"; + + // ---------------------------------------------------------- + // Benchmark Suite + // ---------------------------------------------------------- + m.def("run_benchmark", [](const std::vector& args) { + // Convert Python list of strings to C-style argc/argv + std::vector cstrings; + + // argv[0] is conventionally the program name + cstrings.push_back(const_cast("volestipy_benchmark")); + + for (const auto& s : args) { + cstrings.push_back(const_cast(s.c_str())); + } + + // Call your refactored main function + return run_benchmark_cli(cstrings.size(), cstrings.data()); + + }, py::arg("args") = std::vector(), + R"pbdoc( + Run the C++ benchmark suite. + + Parameters + ---------- + args : list of str + Command line arguments as you would pass them in the terminal. + Example: ["--config", "config.json", "--dim", "10", "--walk", "BallWalk"] + )pbdoc"); } diff --git a/volestipy/__init__.py b/volestipy/__init__.py index b1020e8..5b971b5 100644 --- a/volestipy/__init__.py +++ b/volestipy/__init__.py @@ -74,6 +74,7 @@ def _import_extension(): ess, univariate_psrf, multivariate_psrf, + run_benchmark, ) @@ -651,6 +652,7 @@ def birkhoff_polytope(n: int) -> HPolytope: "ess", "univariate_psrf", "multivariate_psrf", + "run_benchmark", ] __version__ = "0.1.0" From 0091f36d7f5bf33b2e958e2371e19c1ddfdde814 Mon Sep 17 00:00:00 2001 From: Christoforos Date: Sun, 2 Aug 2026 00:32:49 +0300 Subject: [PATCH 2/5] Added tests --- CMakeLists.txt | 2 +- benchmark/run_benchmark.py | 4 +- benchmark/tests/conftest.py | 161 ++++++++++++++++++++++ benchmark/tests/test_config_validation.py | 128 +++++++++++++++++ benchmark/tests/test_integration.py | 74 ++++++++++ benchmark/tests/test_main_flow.py | 103 ++++++++++++++ benchmark/walk_config.json | 1 + external/volesti | 2 +- pyproject.toml | 3 + src/bindings/volesti_bindings.cpp | 4 +- 10 files changed, 476 insertions(+), 6 deletions(-) create mode 100644 benchmark/tests/conftest.py create mode 100644 benchmark/tests/test_config_validation.py create mode 100644 benchmark/tests/test_integration.py create mode 100644 benchmark/tests/test_main_flow.py diff --git a/CMakeLists.txt b/CMakeLists.txt index ac60dbf..054b8cc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -160,7 +160,7 @@ endif() pybind11_add_module(_volestipy MODULE src/bindings/volesti_bindings.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/external/volesti/benchmark/src/benchmark_cli.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/external/volesti/benchmark/src/benchmark_run.cpp ${CMAKE_CURRENT_SOURCE_DIR}/external/volesti/benchmark/src/benchmark_utils.cpp ${CMAKE_CURRENT_SOURCE_DIR}/external/volesti/benchmark/src/walk_parameters.cpp ${CMAKE_CURRENT_SOURCE_DIR}/external/volesti/benchmark/src/walk_registry.cpp diff --git a/benchmark/run_benchmark.py b/benchmark/run_benchmark.py index 58c6288..ff002cd 100644 --- a/benchmark/run_benchmark.py +++ b/benchmark/run_benchmark.py @@ -11,7 +11,7 @@ def main(): try: existing_rows = len(pd.read_csv(csv_file)) except pd.errors.EmptyDataError: - pass # File exists but is empty + pass script_dir = os.path.dirname(os.path.abspath(__file__)) config_path = os.path.abspath(os.path.join(script_dir, "walk_config.json")) @@ -33,7 +33,7 @@ def main(): print("Salvaging data collected so far...\n") finally: - # Pass the row offset so we ONLY plot the new stuff! + # Pass the row offset so we only plot the new stuff plot_results(csv_file, start_row=existing_rows) if __name__ == "__main__": diff --git a/benchmark/tests/conftest.py b/benchmark/tests/conftest.py new file mode 100644 index 0000000..f7dd167 --- /dev/null +++ b/benchmark/tests/conftest.py @@ -0,0 +1,161 @@ +""" +Shared pytest fixtures for the benchmark test suite. + +Key idea: `run_benchmark` in the C++ module has no return value that's +useful to assert on directly (it just writes a CSV as a side effect and +may raise). So most unit tests fake `volestipy.run_benchmark` with a +Python stand-in that mimics its observable behavior (writes rows to the +CSV, or raises), which lets us test main()'s control flow (success / +KeyboardInterrupt / Exception -> always calls plot_results) without +needing the compiled extension or a real, slow sampling run at all. + +The one real end-to-end test (test_integration.py) does call the actual +volestipy.run_benchmark, but with a deliberately tiny/fast config. +""" +import json +import os +import sys +import shutil +import pytest +import pandas as pd + +REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +if REPO_ROOT not in sys.path: + sys.path.insert(0, REPO_ROOT) + + +@pytest.fixture +def tmp_cwd(tmp_path, monkeypatch): + """Run the test inside an isolated temp directory (own cwd), + since main() writes 'benchmark_results.csv' relative to cwd.""" + monkeypatch.chdir(tmp_path) + return tmp_path + + +@pytest.fixture +def base_config(): + """A full config matching your schema, as a Python dict, so tests + can mutate/parametrize individual fields instead of hand-editing JSON.""" + return { + "global_settings": { + "target_ESS": 3000, + "time_limit_sec": 30.0, + "base_seed": 42, + "dimensions": [10, 20, 30, 40, 50, 60], + "rotation_angle": 53, + "polytope_choice": "Cube", + "custom_A_file": "Cube_50_A.csv", + "custom_b_file": "Cube_50_b.csv", + "dynamic_batch_size": True, + "write_to_file": True, + "rounding": False, + "auto_walk": False, + "show_console_logs": False, + "show_menu": False, + }, + "walks": { + "BallWalk": {"enabled": False, "samples": 20000, "walk_len_multiplier": 2, "walk_len_base": 0}, + "BilliardWalk": {"enabled": True, "samples": 2000, "walk_len_multiplier": 0, "walk_len_base": 1}, + "AcceleratedBilliardWalk": {"enabled": True, "samples": 2000, "walk_len_multiplier": 0, "walk_len_base": 1}, + "SparseBilliardWalk": {"enabled": False, "samples": 1000, "walk_len_multiplier": 0, "walk_len_base": 1}, + "CDHRWalk": {"enabled": True, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0}, + "RDHRWalk": {"enabled": False, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0}, + "DikinWalk": {"enabled": False, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0}, + "JohnWalk": {"enabled": False, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0}, + "VaidyaWalk": {"enabled": False, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0}, + "GaussianBallWalk": {"enabled": False, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0, "a_i_param": 1.0}, + "GaussianCDHRWalk": {"enabled": False, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0, "a_i_param": 1.0}, + "BilliardShakeAndBakeWalk": {"enabled": False, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0}, + "ShakeAndBakeWalk": {"enabled": False, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0}, + "BCDHRWalk": {"enabled": False, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0}, + "BRDHRWalk": {"enabled": False, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0}, + "CRHMCWalk": {"enabled": False, "samples": 1000, "walk_len_multiplier": 0, "walk_len_base": 1}, + }, + } + + +@pytest.fixture +def fast_config(): + """A minimal config meant to actually finish in ~1-2s against the + real C++ extension: one tiny dimension, one cheap walk, few samples, + a hard time limit as a safety net.""" + return { + "global_settings": { + "target_ESS": 100, + "time_limit_sec": 5.0, + "base_seed": 1, + "dimensions": [2], + "rotation_angle": 0, + "polytope_choice": "Cube", + "custom_A_file": "", + "custom_b_file": "", + "dynamic_batch_size": True, + "write_to_file": True, + "rounding": False, + "auto_walk": False, + "show_console_logs": False, + "show_menu": False, + }, + "walks": { + "BilliardWalk": {"enabled": True, "samples": 200, "walk_len_multiplier": 0, "walk_len_base": 1}, + }, + } + + +@pytest.fixture +def write_config(tmp_cwd): + """Helper: dump a config dict to walk_config.json in the tmp cwd, + matching the path resolution main() uses (next to the script).""" + def _write(config_dict, name="walk_config.json"): + path = tmp_cwd / name + path.write_text(json.dumps(config_dict, indent=2)) + return str(path) + return _write + + +class FakeVolestipy: + """Drop-in stand-in for the `volestipy` C++ extension module. + + Configure `.behavior` to control what run_benchmark() does: + - "success": writes a couple of fake result rows to the CSV + - "keyboard_interrupt": raises KeyboardInterrupt after writing partial rows + - "crash": raises a RuntimeError after writing partial rows + - "noop": does nothing (simulates e.g. all walks disabled) + """ + def __init__(self, behavior="success", csv_file="benchmark_results.csv"): + self.behavior = behavior + self.csv_file = csv_file + self.calls = [] + + def run_benchmark(self, args): + self.calls.append(list(args)) + + def append_rows(rows): + df = pd.DataFrame(rows) + header = not os.path.exists(self.csv_file) + df.to_csv(self.csv_file, mode="a", header=header, index=False) + + if self.behavior == "noop": + return + if self.behavior == "success": + append_rows([{"walk": "BilliardWalk", "dim": 2, "ess": 150, "time_sec": 0.4}]) + return + if self.behavior == "keyboard_interrupt": + append_rows([{"walk": "BilliardWalk", "dim": 2, "ess": 40, "time_sec": 0.1}]) + raise KeyboardInterrupt() + if self.behavior == "crash": + append_rows([{"walk": "BilliardWalk", "dim": 2, "ess": 10, "time_sec": 0.05}]) + raise RuntimeError("simulated C++ crash (e.g. segfault-adjacent LP failure)") + raise ValueError(f"unknown behavior {self.behavior}") + + +@pytest.fixture +def fake_volestipy_module(monkeypatch): + """Install a FakeVolestipy() into sys.modules['volestipy'] so that + `import volestipy` inside your benchmark script picks it up, and + return the instance so tests can configure .behavior and inspect .calls.""" + def _install(behavior="success"): + fake_module = FakeVolestipy(behavior=behavior) + monkeypatch.setitem(sys.modules, "volestipy", fake_module) + return fake_module + return _install diff --git a/benchmark/tests/test_config_validation.py b/benchmark/tests/test_config_validation.py new file mode 100644 index 0000000..eb7a604 --- /dev/null +++ b/benchmark/tests/test_config_validation.py @@ -0,0 +1,128 @@ +import json +import pytest + +KNOWN_WALKS = { + "BallWalk", "BilliardWalk", "AcceleratedBilliardWalk", "SparseBilliardWalk", + "CDHRWalk", "RDHRWalk", "DikinWalk", "JohnWalk", "VaidyaWalk", + "GaussianBallWalk", "GaussianCDHRWalk", "BilliardShakeAndBakeWalk", + "ShakeAndBakeWalk", "BCDHRWalk", "BRDHRWalk", "CRHMCWalk", +} + +REQUIRED_GLOBAL_KEYS = { + "target_ESS", "time_limit_sec", "base_seed", "dimensions", + "rotation_angle", "polytope_choice", "custom_A_file", "custom_b_file", + "dynamic_batch_size", "write_to_file", "rounding", "auto_walk", + "show_console_logs", "show_menu", +} + +REQUIRED_WALK_KEYS = {"enabled", "samples", "walk_len_multiplier", "walk_len_base"} + +VALID_POLYTOPES = {"Cube", "Custom"} # extend as your C++ side supports more + + +def validate_config(config: dict) -> list[str]: + """Returns a list of human-readable problems; empty list = valid.""" + problems = [] + + gs = config.get("global_settings") + if gs is None: + return ["missing 'global_settings' section"] + + missing_global = REQUIRED_GLOBAL_KEYS - gs.keys() + if missing_global: + problems.append(f"global_settings missing keys: {sorted(missing_global)}") + + if "dimensions" in gs: + dims = gs["dimensions"] + if not isinstance(dims, list) or not dims: + problems.append("dimensions must be a non-empty list") + else: + bad = [d for d in dims if not isinstance(d, int) or d <= 0] + if bad: + problems.append(f"dimensions must be positive ints, got: {bad}") + + if "target_ESS" in gs and (not isinstance(gs["target_ESS"], (int, float)) or gs["target_ESS"] <= 0): + problems.append("target_ESS must be a positive number") + + if "time_limit_sec" in gs and (not isinstance(gs["time_limit_sec"], (int, float)) or gs["time_limit_sec"] <= 0): + problems.append("time_limit_sec must be a positive number") + + if gs.get("polytope_choice") not in VALID_POLYTOPES and "polytope_choice" in gs: + # not fatal-fail this hard in prod, but flag it for tests + problems.append(f"polytope_choice {gs.get('polytope_choice')!r} not in {VALID_POLYTOPES}") + + walks = config.get("walks") + if walks is None: + problems.append("missing 'walks' section") + return problems + + unknown_walks = set(walks.keys()) - KNOWN_WALKS + if unknown_walks: + problems.append(f"unknown walk name(s): {sorted(unknown_walks)}") + + any_enabled = False + for name, wcfg in walks.items(): + missing = REQUIRED_WALK_KEYS - wcfg.keys() + if missing: + problems.append(f"walk '{name}' missing keys: {sorted(missing)}") + continue + if wcfg["enabled"]: + any_enabled = True + if not isinstance(wcfg["samples"], int) or wcfg["samples"] <= 0: + problems.append(f"walk '{name}' samples must be a positive int") + if wcfg["walk_len_multiplier"] == 0 and wcfg["walk_len_base"] == 0: + problems.append(f"walk '{name}' has walk_len_multiplier and walk_len_base both 0 " + f"(walk length would be 0)") + + if not any_enabled: + problems.append("no walk is enabled - benchmark would do nothing") + + return problems + +def test_provided_config_is_valid(base_config): + problems = validate_config(base_config) + assert problems == [] + + +def test_no_enabled_walk_is_flagged(base_config): + for w in base_config["walks"].values(): + w["enabled"] = False + problems = validate_config(base_config) + assert any("no walk is enabled" in p for p in problems) + + +def test_unknown_walk_name_is_flagged(base_config): + base_config["walks"]["TotallyMadeUpWalk"] = { + "enabled": True, "samples": 10, "walk_len_multiplier": 1, "walk_len_base": 0 + } + problems = validate_config(base_config) + assert any("unknown walk" in p for p in problems) + + +@pytest.mark.parametrize("bad_dims", [[], [-5], [0], ["10"], None]) +def test_invalid_dimensions_are_flagged(base_config, bad_dims): + base_config["global_settings"]["dimensions"] = bad_dims + problems = validate_config(base_config) + assert any("dimensions" in p for p in problems) + + +@pytest.mark.parametrize("bad_samples", [0, -1, 3.5, "2000"]) +def test_invalid_samples_flagged(base_config, bad_samples): + base_config["walks"]["BilliardWalk"]["samples"] = bad_samples + problems = validate_config(base_config) + assert any("BilliardWalk" in p and "samples" in p for p in problems) + + +def test_zero_walk_length_flagged(base_config): + base_config["walks"]["BilliardWalk"]["walk_len_multiplier"] = 0 + base_config["walks"]["BilliardWalk"]["walk_len_base"] = 0 + problems = validate_config(base_config) + assert any("walk length would be 0" in p for p in problems) + + +def test_json_file_roundtrips(write_config, base_config): + path = write_config(base_config) + with open(path) as f: + loaded = json.load(f) + assert loaded == base_config + assert validate_config(loaded) == [] diff --git a/benchmark/tests/test_integration.py b/benchmark/tests/test_integration.py new file mode 100644 index 0000000..5e67931 --- /dev/null +++ b/benchmark/tests/test_integration.py @@ -0,0 +1,74 @@ +import os +import pandas as pd +import pytest + +pytest.importorskip("volestipy", reason="compiled volestipy extension not available") +import volestipy + +ALL_WALKS = [ + "BallWalk", "BilliardWalk", "AcceleratedBilliardWalk", "SparseBilliardWalk", + "CDHRWalk", "RDHRWalk", "DikinWalk", "JohnWalk", "VaidyaWalk", + "GaussianBallWalk", "GaussianCDHRWalk", "BilliardShakeAndBakeWalk", + "ShakeAndBakeWalk", "BCDHRWalk", "BRDHRWalk", "CRHMCWalk", +] + +pytestmark = pytest.mark.integration + +def _single_walk_config(walk_name, dim=2, samples=200, extra=None): + walk_cfg = {"enabled": True, "samples": samples, "walk_len_multiplier": 0, "walk_len_base": 1} + if extra: + walk_cfg.update(extra) + return { + "global_settings": { + "target_ESS": 100, + "time_limit_sec": 5.0, + "base_seed": 1, + "dimensions": [dim], + "rotation_angle": 0, + "polytope_choice": "Cube", + "custom_A_file": "", + "custom_b_file": "", + "dynamic_batch_size": True, + "write_to_file": True, + "rounding": False, + "auto_walk": False, + "show_console_logs": False, + "show_menu": False, + }, + "walks": {walk_name: walk_cfg}, + } + + +def test_fast_config_end_to_end(tmp_cwd, write_config, fast_config): + path = write_config(fast_config) + + volestipy.run_benchmark(["--config", path]) + + assert os.path.exists("benchmark_results.csv") + df = pd.read_csv("benchmark_results.csv") + assert len(df) > 0 + + +@pytest.mark.parametrize("walk_name", ALL_WALKS) +def test_each_walk_runs_without_crashing(tmp_cwd, write_config, walk_name): + extra = {"a_i_param": 1.0} if "Gaussian" in walk_name else None + config = _single_walk_config(walk_name, extra=extra) + path = write_config(config) + + volestipy.run_benchmark(["--config", path]) + + assert os.path.exists("benchmark_results.csv") + + +def test_time_limit_is_respected(tmp_cwd, write_config): + import time + config = _single_walk_config("BallWalk", dim=30, samples=10_000) + config["global_settings"]["time_limit_sec"] = 2.0 + config["global_settings"]["target_ESS"] = 5000 + path = write_config(config) + + start = time.time() + volestipy.run_benchmark(["--config", path]) + elapsed = time.time() - start + + assert elapsed < 15.0, f"time_limit_sec=2.0 was not respected, took {elapsed:.1f}s" diff --git a/benchmark/tests/test_main_flow.py b/benchmark/tests/test_main_flow.py new file mode 100644 index 0000000..1b7b8da --- /dev/null +++ b/benchmark/tests/test_main_flow.py @@ -0,0 +1,103 @@ +import importlib +import sys +import types +import pandas as pd +import pytest + +MODULE_NAME = "run_benchmark" + + +@pytest.fixture +def script_module(monkeypatch, fake_volestipy_module): + fake_volestipy_module("success") + + plot_calls = [] + fake_plot_module = types.ModuleType("plot") + + def fake_plot_results(csv_file, start_row=0): + plot_calls.append({"csv_file": csv_file, "start_row": start_row}) + + fake_plot_module.plot_results = fake_plot_results + monkeypatch.setitem(sys.modules, "plot", fake_plot_module) + + sys.modules.pop(MODULE_NAME, None) + mod = importlib.import_module(MODULE_NAME) + mod._plot_calls = plot_calls # stash for assertions + return mod + + +def _get_fake_volestipy(): + return sys.modules["volestipy"] + + +def test_success_path_calls_plot_once_with_start_row_zero(tmp_cwd, write_config, base_config, script_module): + write_config(base_config) + script_module.main() + + assert len(script_module._plot_calls) == 1 + assert script_module._plot_calls[0]["start_row"] == 0 + fake = _get_fake_volestipy() + assert len(fake.calls) == 1 + + +def test_config_path_passed_correctly(tmp_cwd, write_config, base_config, script_module): + """The wrapper builds argv as ["--config", ] - verify that's + actually what reaches run_benchmark, since a relative/wrong path here + would fail deep inside C++ with a much less clear error.""" + write_config(base_config) + script_module.main() + + fake = _get_fake_volestipy() + args = fake.calls[0] + assert "--config" in args + config_arg = args[args.index("--config") + 1] + assert config_arg.endswith("walk_config.json") + import os + assert os.path.isabs(config_arg) + + +def test_keyboard_interrupt_still_plots_and_does_not_propagate(tmp_cwd, write_config, base_config, + fake_volestipy_module, script_module): + fake_volestipy_module("keyboard_interrupt") + write_config(base_config) + + script_module.main() + + assert len(script_module._plot_calls) == 1 + + +def test_crash_still_plots_partial_data(tmp_cwd, write_config, base_config, + fake_volestipy_module, script_module): + fake_volestipy_module("crash") + write_config(base_config) + + script_module.main() + + assert len(script_module._plot_calls) == 1 + df = pd.read_csv("benchmark_results.csv") + assert len(df) == 1 + + +def test_start_row_offset_on_rerun(tmp_cwd, write_config, base_config, + fake_volestipy_module, script_module): + """Run the benchmark twice against an existing CSV - the second call's + plot should be told to start at the row count from the first run, + not re-plot everything from scratch.""" + fake_volestipy_module("success") + write_config(base_config) + + script_module.main() + first_len = len(pd.read_csv("benchmark_results.csv")) + + script_module.main() + assert script_module._plot_calls[-1]["start_row"] == first_len + + +def test_missing_config_file_is_handled_gracefully(tmp_cwd, base_config, script_module): + """Don't write any config file - main() should hit the except branch + (or fail clearly) rather than hanging or crashing pytest itself.""" + fake = _get_fake_volestipy() + fake.behavior = "crash" + + script_module.main() + assert len(script_module._plot_calls) == 1 diff --git a/benchmark/walk_config.json b/benchmark/walk_config.json index fa3448c..b02bb76 100644 --- a/benchmark/walk_config.json +++ b/benchmark/walk_config.json @@ -12,6 +12,7 @@ "write_to_file": true, "rounding": false, "auto_walk": false, + "show_console_logs": false, "show_menu": false }, "walks": { diff --git a/external/volesti b/external/volesti index 6e8191f..f4edcb7 160000 --- a/external/volesti +++ b/external/volesti @@ -1 +1 @@ -Subproject commit 6e8191f2ee52effaf24ed6d3af0cf5c0bd1b87c0 +Subproject commit f4edcb76079b3af44ea46cc1eb5844f37ebd5cfc diff --git a/pyproject.toml b/pyproject.toml index 6ec90b3..52a109e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,3 +39,6 @@ package-dir = {"" = "."} [tool.setuptools.packages.find] where = ["."] include = ["volestipy*"] + +[tool.pytest.ini_options] +markers = ["integration: real end-to-end tests that call the compiled volestipy extension (slower)"] diff --git a/src/bindings/volesti_bindings.cpp b/src/bindings/volesti_bindings.cpp index 49fd49d..4be5687 100644 --- a/src/bindings/volesti_bindings.cpp +++ b/src/bindings/volesti_bindings.cpp @@ -55,7 +55,7 @@ #include "preprocess/inscribed_ellipsoid_rounding.hpp" // For Benchmark -#include "benchmark/include/benchmark_cli.hpp" +#include "benchmark/include/benchmark_run.hpp" #include #include @@ -1017,7 +1017,7 @@ R : float } // Call your refactored main function - return run_benchmark_cli(cstrings.size(), cstrings.data()); + return run_benchmark(cstrings.size(), cstrings.data()); }, py::arg("args") = std::vector(), R"pbdoc( From b59f6d7bb3c99f5f14232f8bd0f12684b3d04ffc Mon Sep 17 00:00:00 2001 From: Christoforos Date: Sun, 2 Aug 2026 00:37:13 +0300 Subject: [PATCH 3/5] Some comment changed --- benchmark/tests/conftest.py | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/benchmark/tests/conftest.py b/benchmark/tests/conftest.py index f7dd167..11ec9ca 100644 --- a/benchmark/tests/conftest.py +++ b/benchmark/tests/conftest.py @@ -1,17 +1,3 @@ -""" -Shared pytest fixtures for the benchmark test suite. - -Key idea: `run_benchmark` in the C++ module has no return value that's -useful to assert on directly (it just writes a CSV as a side effect and -may raise). So most unit tests fake `volestipy.run_benchmark` with a -Python stand-in that mimics its observable behavior (writes rows to the -CSV, or raises), which lets us test main()'s control flow (success / -KeyboardInterrupt / Exception -> always calls plot_results) without -needing the compiled extension or a real, slow sampling run at all. - -The one real end-to-end test (test_integration.py) does call the actual -volestipy.run_benchmark, but with a deliberately tiny/fast config. -""" import json import os import sys @@ -26,16 +12,11 @@ @pytest.fixture def tmp_cwd(tmp_path, monkeypatch): - """Run the test inside an isolated temp directory (own cwd), - since main() writes 'benchmark_results.csv' relative to cwd.""" monkeypatch.chdir(tmp_path) return tmp_path - @pytest.fixture def base_config(): - """A full config matching your schema, as a Python dict, so tests - can mutate/parametrize individual fields instead of hand-editing JSON.""" return { "global_settings": { "target_ESS": 3000, @@ -73,12 +54,8 @@ def base_config(): }, } - @pytest.fixture def fast_config(): - """A minimal config meant to actually finish in ~1-2s against the - real C++ extension: one tiny dimension, one cheap walk, few samples, - a hard time limit as a safety net.""" return { "global_settings": { "target_ESS": 100, @@ -104,8 +81,6 @@ def fast_config(): @pytest.fixture def write_config(tmp_cwd): - """Helper: dump a config dict to walk_config.json in the tmp cwd, - matching the path resolution main() uses (next to the script).""" def _write(config_dict, name="walk_config.json"): path = tmp_cwd / name path.write_text(json.dumps(config_dict, indent=2)) @@ -114,14 +89,6 @@ def _write(config_dict, name="walk_config.json"): class FakeVolestipy: - """Drop-in stand-in for the `volestipy` C++ extension module. - - Configure `.behavior` to control what run_benchmark() does: - - "success": writes a couple of fake result rows to the CSV - - "keyboard_interrupt": raises KeyboardInterrupt after writing partial rows - - "crash": raises a RuntimeError after writing partial rows - - "noop": does nothing (simulates e.g. all walks disabled) - """ def __init__(self, behavior="success", csv_file="benchmark_results.csv"): self.behavior = behavior self.csv_file = csv_file @@ -151,9 +118,6 @@ def append_rows(rows): @pytest.fixture def fake_volestipy_module(monkeypatch): - """Install a FakeVolestipy() into sys.modules['volestipy'] so that - `import volestipy` inside your benchmark script picks it up, and - return the instance so tests can configure .behavior and inspect .calls.""" def _install(behavior="success"): fake_module = FakeVolestipy(behavior=behavior) monkeypatch.setitem(sys.modules, "volestipy", fake_module) From a46341936b5d6efbce83d4af20d109f443f452b0 Mon Sep 17 00:00:00 2001 From: Christoforos Date: Mon, 3 Aug 2026 02:50:22 +0300 Subject: [PATCH 4/5] Minor stuff --- src/bindings/volesti_bindings.cpp | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/bindings/volesti_bindings.cpp b/src/bindings/volesti_bindings.cpp index 4be5687..870e9c3 100644 --- a/src/bindings/volesti_bindings.cpp +++ b/src/bindings/volesti_bindings.cpp @@ -1002,21 +1002,15 @@ R : float m.attr("__version__") = "0.1.0"; m.attr("__volesti_version__") = "1.1.2"; - // ---------------------------------------------------------- - // Benchmark Suite - // ---------------------------------------------------------- + // *****Benchmark Suite***** m.def("run_benchmark", [](const std::vector& args) { - // Convert Python list of strings to C-style argc/argv std::vector cstrings; - // argv[0] is conventionally the program name cstrings.push_back(const_cast("volestipy_benchmark")); for (const auto& s : args) { cstrings.push_back(const_cast(s.c_str())); } - - // Call your refactored main function return run_benchmark(cstrings.size(), cstrings.data()); }, py::arg("args") = std::vector(), From a6c1b1874bfd575a742ca5d9f4c7d66ccd217a11 Mon Sep 17 00:00:00 2001 From: Christoforos Date: Mon, 3 Aug 2026 02:53:42 +0300 Subject: [PATCH 5/5] Minor boring stuff --- benchmark/tests/test_config_validation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/tests/test_config_validation.py b/benchmark/tests/test_config_validation.py index eb7a604..7da679a 100644 --- a/benchmark/tests/test_config_validation.py +++ b/benchmark/tests/test_config_validation.py @@ -17,7 +17,7 @@ REQUIRED_WALK_KEYS = {"enabled", "samples", "walk_len_multiplier", "walk_len_base"} -VALID_POLYTOPES = {"Cube", "Custom"} # extend as your C++ side supports more +VALID_POLYTOPES = {"Cube", "Custom"} def validate_config(config: dict) -> list[str]: @@ -48,7 +48,7 @@ def validate_config(config: dict) -> list[str]: problems.append("time_limit_sec must be a positive number") if gs.get("polytope_choice") not in VALID_POLYTOPES and "polytope_choice" in gs: - # not fatal-fail this hard in prod, but flag it for tests + problems.append(f"polytope_choice {gs.get('polytope_choice')!r} not in {VALID_POLYTOPES}") walks = config.get("walks")