Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .cache/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
clangd/
41 changes: 30 additions & 11 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -108,11 +108,13 @@ endif()

# The S3 benchmark needs the io library (rest backend) and the benchmark tree.
# Checked after the io gating above so a force-disabled io also disables it.
if(CUCASCADE_BUILD_S3_BENCHMARK AND (NOT CUCASCADE_BUILD_IO
OR NOT CUCASCADE_BUILD_BENCHMARKS))
if(CUCASCADE_BUILD_S3_BENCHMARK
AND (NOT CUCASCADE_BUILD_IO
OR NOT CUCASCADE_BUILD_BENCHMARKS
OR NOT CUCASCADE_BUILD_CUDF))
message(
STATUS
"CUCASCADE_BUILD_S3_BENCHMARK disabled: requires CUCASCADE_BUILD_IO=ON and CUCASCADE_BUILD_BENCHMARKS=ON"
"CUCASCADE_BUILD_S3_BENCHMARK disabled: requires CUCASCADE_BUILD_IO=ON, CUCASCADE_BUILD_BENCHMARKS=ON and CUCASCADE_BUILD_CUDF=ON (it links cucascade_cudf and kvikIO)"
)
set(CUCASCADE_BUILD_S3_BENCHMARK
OFF
Expand Down Expand Up @@ -158,10 +160,15 @@ if(NOT CUCASCADE_TOPOLOGY_ONLY)
pkg_check_modules(CURL REQUIRED IMPORTED_TARGET libcurl)
find_package(OpenSSL REQUIRED)

# kvikIO — backs the local-file fallback ioctx (kvikio_context). Used
# directly (not via cudf) so the io library stays cudf-free. Not swappable:
# unlike moodycamel/invocable below there is no in-tree stand-in to replace.
find_package(kvikio REQUIRED CONFIG)
# kvikIO — backs the local-file fallback ioctx (kvikio_context). It reaches
# the environment only through libcudf's dependency closure (libkvikio is
# not a direct dependency), so it is tied to CUCASCADE_BUILD_CUDF: a
# cudf-free build drops kvikio_context and its catch-all registry entry,
# leaving uring/restful to claim paths. Consumers see the difference via the
# CUCASCADE_HAS_KVIKIO definition propagated by cucascade_io_thirdparty.
if(CUCASCADE_BUILD_CUDF)
find_package(kvikio REQUIRED CONFIG)
endif()

# cucascade_io_thirdparty carries the swappable moodycamel + invocable
# (abseil) usage requirements from a single place; the io object library,
Expand Down Expand Up @@ -239,6 +246,15 @@ if(NOT CUCASCADE_TOPOLOGY_ONLY)
target_compile_definitions(cucascade_io_thirdparty
INTERFACE CUCASCADE_USE_ABSEIL_INVOCABLE)
endif()

# Gates the kvikIO fallback ioctx in io_config and the datasource registry.
# Carried on the same INTERFACE target as the definitions above so the io
# object library, its installable variants, and installed consumers all
# agree on the layout of io_config.
if(CUCASCADE_BUILD_CUDF)
target_compile_definitions(cucascade_io_thirdparty
INTERFACE CUCASCADE_HAS_KVIKIO)
endif()
endif()

# Find numa (provided by numactl-devel or libnuma-dev depending on the package
Expand Down Expand Up @@ -407,7 +423,10 @@ if(CUCASCADE_BUILD_IO)
# side by cuCascadeConfig.cmake (same names), mirroring the Numa::Numa
# approach.
set(CUCASCADE_IO_LINK_LIBS PkgConfig::LIBURING PkgConfig::CURL
OpenSSL::Crypto kvikio::kvikio)
OpenSSL::Crypto)
if(CUCASCADE_BUILD_CUDF)
list(APPEND CUCASCADE_IO_LINK_LIBS kvikio::kvikio)
endif()

target_link_libraries(
cucascade_io_objects PUBLIC cucascade_objects ${CUCASCADE_IO_LINK_LIBS}
Expand All @@ -425,7 +444,7 @@ endif()
target_include_directories(cucascade_topology_discovery_objects
PUBLIC ${CUCASCADE_PUBLIC_INCLUDE_DIRS})
target_link_libraries(cucascade_topology_discovery_objects
PUBLIC CUDA::nvml_static rmm::rmm)
PUBLIC CUDA::nvml_static CUDA::cuda_driver)
target_compile_features(cucascade_topology_discovery_objects PUBLIC cxx_std_20)
target_compile_features(cucascade_topology_discovery_objects
PRIVATE cuda_std_20)
Expand All @@ -443,7 +462,7 @@ if(CUCASCADE_BUILD_STATIC_LIBS)
cucascade_topology_discovery_static)

target_link_libraries(cucascade_topology_discovery_static
PRIVATE CUDA::nvml_static rmm::rmm)
PRIVATE CUDA::nvml_static CUDA::cuda_driver)
target_include_directories(cucascade_topology_discovery_static
PUBLIC ${CUCASCADE_PUBLIC_INCLUDE_DIRS})
target_compile_features(cucascade_topology_discovery_static PUBLIC cxx_std_20)
Expand Down Expand Up @@ -521,7 +540,7 @@ if(CUCASCADE_BUILD_SHARED_LIBS)
cucascade_topology_discovery_shared)

target_link_libraries(cucascade_topology_discovery_shared
PRIVATE CUDA::nvml_static rmm::rmm)
PRIVATE CUDA::nvml_static CUDA::cuda_driver)
target_include_directories(cucascade_topology_discovery_shared
PUBLIC ${CUCASCADE_PUBLIC_INCLUDE_DIRS})
target_compile_features(cucascade_topology_discovery_shared PUBLIC cxx_std_20)
Expand Down
8 changes: 7 additions & 1 deletion include/cucascade/io/config.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,9 @@
#pragma once

#include <cucascade/io/cache/config.hpp>
#ifdef CUCASCADE_HAS_KVIKIO
#include <cucascade/io/kvikio/config.hpp>
#endif
#include <cucascade/io/object_store_config.hpp>
#include <cucascade/io/rest/config.hpp>
#include <cucascade/io/uring/config.hpp>
Expand All @@ -35,7 +37,9 @@ namespace cucascade::io {
* Sub-configs:
* - @c local — uring reactor tunables (local-disk IO path).
* - @c rest — REST reactor tunables (S3/object-store IO path).
* - @c kvikio — kvikIO fallback tunables (local-disk catch-all path).
* - @c kvikio — kvikIO fallback tunables (local-disk catch-all path); present
* only when the library is built with CUCASCADE_BUILD_CUDF, which is what
* supplies kvikIO.
* - @c cache — prefetching cache tunables.
* - @c object_store — object-store credentials and endpoint.
*/
Expand All @@ -59,11 +63,13 @@ struct io_config {
/// retry policy, etc.
rest::config rest{};

#ifdef CUCASCADE_HAS_KVIKIO
/// kvikIO fallback configuration — thread-pool size, task/bounce sizing,
/// O_DIRECT, compat mode. All fields default to "unset", leaving kvikIO's
/// own env-var-seeded defaults in place. Note these are process-global once
/// applied; see @ref kvikio_config.
kvikio_config kvikio{};
#endif

/// Prefetching cache configuration — in-flight budget, pool sizing,
/// dispose-after-use policy.
Expand Down
50 changes: 48 additions & 2 deletions include/cucascade/memory/topology_discovery.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,18 @@

namespace cucascade::memory {

/**
* @brief GPU runtime attributes.
*
* Attributes whose discovery requires initializing a CUDA context (via the CUDA
* driver API). Populated only when `discover()` / `discover_runtime_attributes()`
* are explicitly asked to. Kept separate so passive topology discovery via NVML
* and sysfs never has to spin up a CUDA context.
*/
struct gpu_runtime_attributes {
bool hw_decomp{false}; ///< Hardware-accelerated decompression engine present.
};

/**
* @brief GPU information.
*/
Expand All @@ -24,7 +36,9 @@ struct gpu_topology_info {
std::vector<int> cpu_cores; ///< List of CPU core IDs.
std::vector<int> memory_binding; ///< NUMA nodes for memory binding.
std::vector<std::string> network_devices; ///< Network devices (NICs) optimal for this GPU.
bool hw_decompression_available{false}; ///< Hardware-accelerated decompression engine present.
std::optional<gpu_runtime_attributes>
runtime_attributes; ///< Runtime attributes (populated only when explicitly requested; empty
///< means "not queried", not "unsupported").
};

/**
Expand Down Expand Up @@ -110,11 +124,43 @@ class topology_discovery {
* This method performs the actual discovery of GPUs, NUMA nodes, CPU affinity,
* and network devices. It must be called before `get_topology()`.
*
* By default this call uses only NVML and Linux sysfs and therefore does not
* touch the CUDA driver. Set @p with_runtime_attributes to true to also
* populate per-hardware runtime attributes (e.g. `gpu_runtime_attributes`),
* which queries CUDA driver device attributes.
*
* @param net_verification Controls how strictly network devices are validated.
* @param with_runtime_attributes If true, also discover runtime attributes for
* each hardware class (see `discover_runtime_attributes`). Defaults to false so
* that discovery does not touch the CUDA driver. When true, the caller must
* have already initialized the CUDA driver API (see
* `discover_runtime_attributes` for the exact precondition).
* @return true if discovery was successful, false otherwise.
*/
[[nodiscard]] bool discover(
NetworkDeviceVerification net_verification = NetworkDeviceVerification::EXISTS_ACTIVE_IP);
NetworkDeviceVerification net_verification = NetworkDeviceVerification::EXISTS_ACTIVE_IP,
bool with_runtime_attributes = false);

/**
* @brief Discover runtime attributes for each hardware class in @p topology.
*
* Populates the `runtime_attributes` field of each entry in `topology.gpus`
* (and, in the future, other hardware classes). This is the only path in this
* component that issues CUDA driver calls — every other discovery step relies
* solely on NVML and sysfs.
*
* Safe to call multiple times; existing runtime attribute values are
* overwritten.
*
* @pre The CUDA driver API has already been initialized by the caller —
* either via an explicit `cuInit(0)` or via any prior CUDA runtime call that
* transitively initializes the driver. This function does not call `cuInit`
* and does not create a CUDA context; per-GPU queries that fail (e.g.
* because the driver is uninitialized) leave `hw_decomp` as `false`.
*
* @param topology Topology to enrich in place.
*/
static void discover_runtime_attributes(system_topology_info& topology);

/**
* @brief Get the discovered topology information.
Expand Down
8 changes: 7 additions & 1 deletion src/io/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,16 @@ target_sources(
${CMAKE_CURRENT_SOURCE_DIR}/rest/rest_reactor.cpp
${CMAKE_CURRENT_SOURCE_DIR}/uring/uring_ioctx.cpp
${CMAKE_CURRENT_SOURCE_DIR}/uring/uring_reactor.cpp
${CMAKE_CURRENT_SOURCE_DIR}/kvikio/kvikio_context.cpp
${CMAKE_CURRENT_SOURCE_DIR}/rest/s3/sigv4.cpp
${CMAKE_CURRENT_SOURCE_DIR}/rest/s3/sigv4_authorizer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/rest/s3/list_parser.cpp
${CMAKE_CURRENT_SOURCE_DIR}/cache/types.cpp
${CMAKE_CURRENT_SOURCE_DIR}/cache/metadata_store.cpp
${CMAKE_CURRENT_SOURCE_DIR}/cache/prefetching_cache.cpp)

# kvikIO reaches the environment only via libcudf's dependency closure, so the
# fallback ioctx is built only alongside the cudf layer.
if(CUCASCADE_BUILD_CUDF)
target_sources(cucascade_io_objects
PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/kvikio/kvikio_context.cpp)
endif()
9 changes: 8 additions & 1 deletion src/io/datasource_factory.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,9 @@
#include <cucascade/io/config.hpp>
#include <cucascade/io/datasource_factory.hpp>
#include <cucascade/io/io_context.hpp>
#ifdef CUCASCADE_HAS_KVIKIO
#include <cucascade/io/kvikio/kvikio_context.hpp>
#endif
#include <cucascade/io/object_store_config.hpp>
#include <cucascade/io/rest/rest_ioctx.hpp>
#include <cucascade/io/rest/s3/sigv4_authorizer.hpp>
Expand Down Expand Up @@ -78,6 +80,7 @@ std::shared_ptr<rest::request_authorizer> make_s3_authorizer(const object_store_
using scheme_checker_type = io_context_registry::scheme_checker_type;
using factory_type = io_context_registry::factory_type;

#ifdef CUCASCADE_HAS_KVIKIO
factory_type make_kvikio_ioctx_factory()
{
return [](const io_config& config) -> std::shared_ptr<ioctx> {
Expand All @@ -91,6 +94,7 @@ factory_type make_kvikio_ioctx_factory()
}
};
}
#endif

factory_type make_uring_ioctx_factory(
cucascade::memory::memory_reservation_manager& reservation_manager)
Expand Down Expand Up @@ -160,11 +164,14 @@ io_context_registry::io_context_registry(
// uring / rest claim paths via their reactor's static supports() (local
// files and s3:// URLs respectively). kvikio is the universal fallback —
// it can open any local path — so it matches everything and lookup_path
// defers it behind the explicit backends.
// defers it behind the explicit backends. Without kvikIO (a cudf-free
// build) there is no catch-all and unmatched paths resolve to nothing.
#ifdef CUCASCADE_HAS_KVIKIO
_entries.emplace(
io_context_type::kvikio,
entry{
io_context_type::kvikio, [](std::string_view) { return true; }, make_kvikio_ioctx_factory()});
#endif
_entries.emplace(io_context_type::uring,
entry{io_context_type::uring,
&uring::uring_reactor::supports,
Expand Down
Loading