Benchmark suite

.gitignore

@@ -44,3 +44,9 @@ Thumbs.db
 htmlcov/
 .tox/
 external/lp_solve_5.5/
+Volesti-env/
+test.py
+*.tar.gz
+benchmark/__pycache__/
+benchmark/*.csv
+benchmark/results/

CMakeLists.txt

@@ -34,6 +34,15 @@ if(NOT pybind11_FOUND)
     find_package(pybind11 REQUIRED HINTS "${pybind11_DIR}")
 endif()
 
+# --- QD library ---
+find_library(QD_LIB qd)
+
+if(NOT QD_LIB)
+    message(FATAL_ERROR "Could not find QD library")
+endif()
+
+message(STATUS "QD library: ${QD_LIB}")
+
 # --- Find Eigen3 ---
 find_package(Eigen3 3.3 REQUIRED NO_MODULE)
 if(NOT Eigen3_FOUND)
@@ -52,7 +61,7 @@ endif()
 
 # --- Find Boost ---
 # volesti only needs Boost headers (random, math) - no compiled libraries required
-find_package(Boost 1.56 REQUIRED)
+find_package(Boost 1.56 REQUIRED COMPONENTS program_options)
 if(NOT Boost_FOUND)
     # Fallback: locate headers manually
     find_path(Boost_INCLUDE_DIRS NAMES boost/random.hpp
@@ -150,23 +159,41 @@ endif()
 # --- Build the extension ---
 pybind11_add_module(_volestipy MODULE
     src/bindings/volesti_bindings.cpp
+
+    ${CMAKE_CURRENT_SOURCE_DIR}/external/volesti/benchmark/src/benchmark_run.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/external/volesti/benchmark/src/benchmark_utils.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/external/volesti/benchmark/src/walk_parameters.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/external/volesti/benchmark/src/walk_registry.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/external/volesti/benchmark/src/walk_result.cpp
+
     ${LPSOLVE_SOURCES}
 )
 
 target_include_directories(_volestipy PRIVATE
     ${CMAKE_CURRENT_SOURCE_DIR}/include   # project local headers (sampling_minimal.hpp)
     ${VOLESTI_INCLUDE_DIR}
+    ${CMAKE_CURRENT_SOURCE_DIR}/external/volesti
     ${VOLESTI_INCLUDE_DIR}/../external   # Spectra, minimum_ellipsoid, etc.
+    ${VOLESTI_INCLUDE_DIR}/generators
+    ${VOLESTI_INCLUDE_DIR}/include/generators
+    ${VOLESTI_INCLUDE_DIR}/include/preprocess
+    ${VOLESTI_INCLUDE_DIR}/preprocess
+    ${VOLESTI_INCLUDE_DIR}/benchmark/include
+    ${CMAKE_CURRENT_SOURCE_DIR}/external/volesti/benchmark/include
     ${Boost_INCLUDE_DIRS}
     ${LPSOLVE_INCLUDE_DIRS}
 )
 
 target_link_libraries(_volestipy PRIVATE
     Eigen3::Eigen
+    Boost::program_options
+    ${QD_LIB}
+    m
 )
 
 target_compile_definitions(_volestipy PRIVATE
     ${LPSOLVE_DEFINITIONS}
+    DISABLE_NLP_ORACLES
 )
 
 target_compile_options(_volestipy PRIVATE

README.md

@@ -66,7 +66,7 @@ sudo apt-get install -y \
 ### 3. Install Python dependencies
 
 ```bash
-pip install pybind11 numpy
+pip install pybind11 "numpy<2.0"
 ```
 
 ### 4a. Build via `pip` (recommended)

benchmark/available_choices.md

@@ -0,0 +1,54 @@
+# Available Choices
+
+## Target ESS
+
+The code will use a dynamic batch size in order to sample the necessary number of samples needed to reach the targert ESS
+
+## Time limit
+
+The maximum time the code runs. If the time runs out, the code will stop at the first available opportunity.
+
+## Base seed
+
+The seed used by the random number generator. Use any number you like.
+
+## Dimension
+
+The number of dimesnion the generated polytope will have. This only applies to polytopes generated by the generator functions. It is ignored for custom polytopes.
+
+## Rotation angle
+
+You can rotate the input polytope by this number over all pairs of dimensions. Angle is in degrees. Use 0 to avoid rotation.
+
+## Polytope Choice
+
+Choose the polytope that will be sampled from.
+Available polytope choices are: Cube, Simplex, Birkhoff, Cross, OrderPolytope, Custom
+
+## Custom A file and Custom b file
+
+These are the names of the A and b files of your custom polytope defining the polytope Ax<=b. Csv files are expected, with no labels.
+Place them inside build folder next to the executable.  
+
+## Dynamic batch size
+
+If you have this choice to true, the code will run a dynamic batch size in order to hit target ESS.
+If you have this choice to false, the code will generate exactly "samples" amount of samples, the parameter defined for each walk method.
+
+## Write to file
+
+Set this to true in order to create txt files holding all the generated samples. Useful for later use of the samples.
+
+## Rounding
+
+If this is set to true, the code will firrst call a rounding function on the input polytope, the john position.
+The sampling will take place on the rounded polytope before the sample are reverted back to the original and returned.
+
+## Walks
+
+For each walk, you can change these parameters:
+
+- enabled: This will determine if this method will run or not when sampling
+- samples: How many samples the code will take if dynamic batch size is also false. If dynamic batch size is on, target ESS will be used instead to determine when the execution stops.
+- walk_len: It is a thinning parameter. The code will save the sample only after walk_len samples are sampled.
+walk_len = walk_base + dim * walk_multiplier"

benchmark/plot.py

@@ -0,0 +1,53 @@
+import pandas as pd
+import matplotlib.pyplot as plt
+import os
+
+def plot_results(csv_file, start_row=0):
+    """Reads the benchmark CSV and plots data starting from start_row."""
+    if not os.path.exists(csv_file):
+        print(f"Error: {csv_file} was not generated or found.")
+        return
+
+    df = pd.read_csv(csv_file, skipinitialspace=True)
+    df.columns = df.columns.str.strip()
+
+    if start_row > 0:
+        df = df.iloc[start_row:]
+
+    if df.empty:
+        print("No new data to plot.")
+        return
+
+    print(f"Plotting {len(df)} new rows of data...")
+
+    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 6))
+
+    for method_name, method_data in df.groupby('Method'):
+        method_data = method_data.sort_values(by='Dimension')
+
+        ax1.plot(method_data['Dimension'], method_data['Time_Sec'], 
+                 marker='o', label=method_name.strip())
+
+        ax2.plot(method_data['Dimension'], method_data['Mixing_Ratio'], 
+                 marker='o', label=method_name.strip())
+
+    ax1.set_yscale('log')
+    ax1.set_title('Time vs. Dimension')
+    ax1.set_xlabel('Dimension')
+    ax1.set_ylabel('Total Algorithm Time (Seconds)')
+    ax1.grid(True, which="both", linestyle='--', alpha=0.7)
+    ax1.legend()
+
+    ax2.set_yscale('log')
+    ax2.set_title('Mixing Ratio vs. Dimension')
+    ax2.set_xlabel('Dimension')
+    ax2.set_ylabel('Mixing Ratio (Steps / ESS)')
+    ax2.grid(True, which="both", linestyle='--', alpha=0.7)
+    ax2.legend()
+
+    plt.tight_layout()
+    plt.show()
+
+if __name__ == "__main__":
+    # If run directly, plot everything (start_row=0)
+    plot_results("benchmark_results.csv")

benchmark/run_benchmark.py

@@ -0,0 +1,40 @@
+import volestipy
+import os
+import pandas as pd
+from plot import plot_results
+
+def main():
+    csv_file = "benchmark_results.csv"
+
+    existing_rows = 0
+    if os.path.exists(csv_file):
+        try:
+            existing_rows = len(pd.read_csv(csv_file))
+        except pd.errors.EmptyDataError:
+            pass
+
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    config_path = os.path.abspath(os.path.join(script_dir, "walk_config.json"))
+
+    print("\n--- Starting C++ Benchmark ---")
+
+    try:
+        volestipy.run_benchmark([
+            "--config", config_path
+        ])
+        print("--- C++ Benchmark Finished Successfully ---\n")
+
+    except KeyboardInterrupt:
+        print("\n\n!!! Benchmark Interrupted by User (Ctrl+C) !!!")
+        print("Salvaging data collected so far...\n")
+
+    except Exception as e:
+        print(f"\n\n!!! Benchmark Crashed with Error: {e} !!!")
+        print("Salvaging data collected so far...\n")
+
+    finally:
+        # Pass the row offset so we only plot the new stuff
+        plot_results(csv_file, start_row=existing_rows)
+
+if __name__ == "__main__":
+    main()

benchmark/tests/conftest.py

@@ -0,0 +1,125 @@
+import json
+import os
+import sys
+import shutil
+import pytest
+import pandas as pd
+
+REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+if REPO_ROOT not in sys.path:
+    sys.path.insert(0, REPO_ROOT)
+
+
+@pytest.fixture
+def tmp_cwd(tmp_path, monkeypatch):
+    monkeypatch.chdir(tmp_path)
+    return tmp_path
+
+@pytest.fixture
+def base_config():
+    return {
+        "global_settings": {
+            "target_ESS": 3000,
+            "time_limit_sec": 30.0,
+            "base_seed": 42,
+            "dimensions": [10, 20, 30, 40, 50, 60],
+            "rotation_angle": 53,
+            "polytope_choice": "Cube",
+            "custom_A_file": "Cube_50_A.csv",
+            "custom_b_file": "Cube_50_b.csv",
+            "dynamic_batch_size": True,
+            "write_to_file": True,
+            "rounding": False,
+            "auto_walk": False,
+            "show_console_logs": False,
+            "show_menu": False,
+        },
+        "walks": {
+            "BallWalk": {"enabled": False, "samples": 20000, "walk_len_multiplier": 2, "walk_len_base": 0},
+            "BilliardWalk": {"enabled": True, "samples": 2000, "walk_len_multiplier": 0, "walk_len_base": 1},
+            "AcceleratedBilliardWalk": {"enabled": True, "samples": 2000, "walk_len_multiplier": 0, "walk_len_base": 1},
+            "SparseBilliardWalk": {"enabled": False, "samples": 1000, "walk_len_multiplier": 0, "walk_len_base": 1},
+            "CDHRWalk": {"enabled": True, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0},
+            "RDHRWalk": {"enabled": False, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0},
+            "DikinWalk": {"enabled": False, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0},
+            "JohnWalk": {"enabled": False, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0},
+            "VaidyaWalk": {"enabled": False, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0},
+            "GaussianBallWalk": {"enabled": False, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0, "a_i_param": 1.0},
+            "GaussianCDHRWalk": {"enabled": False, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0, "a_i_param": 1.0},
+            "BilliardShakeAndBakeWalk": {"enabled": False, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0},
+            "ShakeAndBakeWalk": {"enabled": False, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0},
+            "BCDHRWalk": {"enabled": False, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0},
+            "BRDHRWalk": {"enabled": False, "samples": 1000, "walk_len_multiplier": 2, "walk_len_base": 0},
+            "CRHMCWalk": {"enabled": False, "samples": 1000, "walk_len_multiplier": 0, "walk_len_base": 1},
+        },
+    }
+
+@pytest.fixture
+def fast_config():
+    return {
+        "global_settings": {
+            "target_ESS": 100,
+            "time_limit_sec": 5.0,
+            "base_seed": 1,
+            "dimensions": [2],
+            "rotation_angle": 0,
+            "polytope_choice": "Cube",
+            "custom_A_file": "",
+            "custom_b_file": "",
+            "dynamic_batch_size": True,
+            "write_to_file": True,
+            "rounding": False,
+            "auto_walk": False,
+            "show_console_logs": False,
+            "show_menu": False,
+        },
+        "walks": {
+            "BilliardWalk": {"enabled": True, "samples": 200, "walk_len_multiplier": 0, "walk_len_base": 1},
+        },
+    }
+
+
+@pytest.fixture
+def write_config(tmp_cwd):
+    def _write(config_dict, name="walk_config.json"):
+        path = tmp_cwd / name
+        path.write_text(json.dumps(config_dict, indent=2))
+        return str(path)
+    return _write
+
+
+class FakeVolestipy:
+    def __init__(self, behavior="success", csv_file="benchmark_results.csv"):
+        self.behavior = behavior
+        self.csv_file = csv_file
+        self.calls = []
+
+    def run_benchmark(self, args):
+        self.calls.append(list(args))
+
+        def append_rows(rows):
+            df = pd.DataFrame(rows)
+            header = not os.path.exists(self.csv_file)
+            df.to_csv(self.csv_file, mode="a", header=header, index=False)
+
+        if self.behavior == "noop":
+            return
+        if self.behavior == "success":
+            append_rows([{"walk": "BilliardWalk", "dim": 2, "ess": 150, "time_sec": 0.4}])
+            return
+        if self.behavior == "keyboard_interrupt":
+            append_rows([{"walk": "BilliardWalk", "dim": 2, "ess": 40, "time_sec": 0.1}])
+            raise KeyboardInterrupt()
+        if self.behavior == "crash":
+            append_rows([{"walk": "BilliardWalk", "dim": 2, "ess": 10, "time_sec": 0.05}])
+            raise RuntimeError("simulated C++ crash (e.g. segfault-adjacent LP failure)")
+        raise ValueError(f"unknown behavior {self.behavior}")
+
+
+@pytest.fixture
+def fake_volestipy_module(monkeypatch):
+    def _install(behavior="success"):
+        fake_module = FakeVolestipy(behavior=behavior)
+        monkeypatch.setitem(sys.modules, "volestipy", fake_module)
+        return fake_module
+    return _install