Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ struct policy_hub
{
using ValueT = it_value_t<InputIteratorT>;

struct Policy500 : ChainedPolicy<500, Policy500, Policy500>
struct Policy500 : detail::chained_policy<500, Policy500, Policy500>
{
using AdjacentDifferencePolicy =
agent_adjacent_difference_policy<128,
Expand Down
2 changes: 1 addition & 1 deletion cub/cub/device/dispatch/tuning/tuning_for.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ struct policy_selector
// TODO(bgruber): remove once we publish the tuning API
struct policy_hub_t
{
struct policy_500_t : ChainedPolicy<500, policy_500_t, policy_500_t>
struct policy_500_t : detail::chained_policy<500, policy_500_t, policy_500_t>
{
using for_policy_t = policy_t<256, 2>;
};
Expand Down
6 changes: 3 additions & 3 deletions cub/cub/device/dispatch/tuning/tuning_histogram.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -214,15 +214,15 @@ struct policy_hub
}

// SM50
struct Policy500 : ChainedPolicy<500, Policy500, Policy500>
struct Policy500 : detail::chained_policy<500, Policy500, Policy500>
{
// TODO This might be worth it to separate usual histogram and the multi one
using AgentHistogramPolicyT =
agent_histogram_policy<384, t_scale(16), BLOCK_LOAD_DIRECT, LOAD_LDG, true, SMEM, false>;
};

// SM90
struct Policy900 : ChainedPolicy<900, Policy900, Policy500>
struct Policy900 : detail::chained_policy<900, Policy900, Policy500>
{
// Use values from tuning if a specialization exists, otherwise pick Policy500
template <typename Tuning>
Expand All @@ -245,7 +245,7 @@ struct policy_hub
static constexpr int init_kernel_pdl_trigger_max_bins = 2048;
};

struct Policy1000 : ChainedPolicy<1000, Policy1000, Policy900>
struct Policy1000 : detail::chained_policy<1000, Policy1000, Policy900>
{
// Use values from tuning if a specialization exists, otherwise pick Policy900
template <typename Tuning>
Expand Down
6 changes: 3 additions & 3 deletions cub/cub/device/dispatch/tuning/tuning_merge_sort.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ struct policy_hub
{
using KeyT = it_value_t<KeyIteratorT>;

struct Policy500 : ChainedPolicy<500, Policy500, Policy500>
struct Policy500 : detail::chained_policy<500, Policy500, Policy500>
{
using MergeSortPolicy =
agent_merge_sort_policy<256,
Expand All @@ -116,7 +116,7 @@ struct policy_hub
#if defined(_NVHPC_CUDA)
using Policy520 = Policy500;
#else
struct Policy520 : ChainedPolicy<520, Policy520, Policy500>
struct Policy520 : detail::chained_policy<520, Policy520, Policy500>
{
using MergeSortPolicy =
agent_merge_sort_policy<512,
Expand All @@ -127,7 +127,7 @@ struct policy_hub
};
#endif

struct Policy600 : ChainedPolicy<600, Policy600, Policy520>
struct Policy600 : detail::chained_policy<600, Policy600, Policy520>
{
using MergeSortPolicy =
agent_merge_sort_policy<256,
Expand Down
16 changes: 8 additions & 8 deletions cub/cub/device/dispatch/tuning/tuning_radix_sort.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -1037,7 +1037,7 @@ struct policy_hub
//------------------------------------------------------------------------------

/// SM50
struct Policy500 : ChainedPolicy<500, Policy500, Policy500>
struct Policy500 : detail::chained_policy<500, Policy500, Policy500>
{
static constexpr int PRIMARY_RADIX_BITS = (sizeof(KeyT) > 1) ? 7 : 5; // 3.5B 32b keys/s, 1.92B 32b pairs/s (TitanX)
static constexpr int SINGLE_TILE_RADIX_BITS = (sizeof(KeyT) > 1) ? 6 : 5;
Expand Down Expand Up @@ -1129,7 +1129,7 @@ struct policy_hub
};

/// SM60 (GP100)
struct Policy600 : ChainedPolicy<600, Policy600, Policy500>
struct Policy600 : detail::chained_policy<600, Policy600, Policy500>
{
static constexpr int PRIMARY_RADIX_BITS = (sizeof(KeyT) > 1) ? 7 : 5; // 6.9B 32b keys/s (Quadro P100)
static constexpr int SINGLE_TILE_RADIX_BITS = (sizeof(KeyT) > 1) ? 6 : 5;
Expand Down Expand Up @@ -1222,7 +1222,7 @@ struct policy_hub
};

/// SM61 (GP104)
struct Policy610 : ChainedPolicy<610, Policy610, Policy600>
struct Policy610 : detail::chained_policy<610, Policy610, Policy600>
{
static constexpr int PRIMARY_RADIX_BITS = (sizeof(KeyT) > 1) ? 7 : 5; // 3.4B 32b keys/s, 1.83B 32b pairs/s (1080)
static constexpr int SINGLE_TILE_RADIX_BITS = (sizeof(KeyT) > 1) ? 6 : 5;
Expand Down Expand Up @@ -1315,7 +1315,7 @@ struct policy_hub
};

/// SM62 (Tegra, less RF)
struct Policy620 : ChainedPolicy<620, Policy620, Policy610>
struct Policy620 : detail::chained_policy<620, Policy620, Policy610>
{
static constexpr int PRIMARY_RADIX_BITS = 5;
static constexpr int ALT_RADIX_BITS = PRIMARY_RADIX_BITS - 1;
Expand Down Expand Up @@ -1390,7 +1390,7 @@ struct policy_hub
};

/// SM70 (GV100)
struct Policy700 : ChainedPolicy<700, Policy700, Policy620>
struct Policy700 : detail::chained_policy<700, Policy700, Policy620>
{
static constexpr int PRIMARY_RADIX_BITS = (sizeof(KeyT) > 1) ? 7 : 5; // 7.62B 32b keys/s (GV100)
static constexpr int SINGLE_TILE_RADIX_BITS = (sizeof(KeyT) > 1) ? 6 : 5;
Expand Down Expand Up @@ -1484,7 +1484,7 @@ struct policy_hub
};

/// SM80
struct Policy800 : ChainedPolicy<800, Policy800, Policy700>
struct Policy800 : detail::chained_policy<800, Policy800, Policy700>
{
static constexpr int PRIMARY_RADIX_BITS = (sizeof(KeyT) > 1) ? 7 : 5;
static constexpr int SINGLE_TILE_RADIX_BITS = (sizeof(KeyT) > 1) ? 6 : 5;
Expand Down Expand Up @@ -1699,12 +1699,12 @@ struct policy_hub
};

struct Policy900
: ChainedPolicy<900, Policy900, Policy800>
: detail::chained_policy<900, Policy900, Policy800>
, OnesweepSmallKeyTunedPolicy<sm90_small_key_tuning<sizeof(KeyT), KEYS_ONLY ? 0 : sizeof(ValueT), sizeof(OffsetT)>>
{};

struct Policy1000
: ChainedPolicy<1000, Policy1000, Policy900>
: detail::chained_policy<1000, Policy1000, Policy900>
, OnesweepSmallKeyTunedPolicy<
sm100_small_key_tuning<ValueT, sizeof(KeyT), KEYS_ONLY ? 0 : sizeof(ValueT), sizeof(OffsetT)>>
{};
Expand Down
6 changes: 3 additions & 3 deletions cub/cub/device/dispatch/tuning/tuning_reduce.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ get_sm100_tuning(type_t accum_t, op_kind_t operation_t, int offset_size, int acc
template <typename AccumT, typename OffsetT, typename ReductionOpT>
struct policy_hub
{
struct Policy500 : ChainedPolicy<500, Policy500, Policy500>
struct Policy500 : detail::chained_policy<500, Policy500, Policy500>
{
static constexpr int threads_per_block = 256;
static constexpr int items_per_thread = 20;
Expand Down Expand Up @@ -279,7 +279,7 @@ struct policy_hub
NoScaling<ReducePolicy::BLOCK_THREADS, ReducePolicy::ITEMS_PER_THREAD>>;
};

struct Policy600 : ChainedPolicy<600, Policy600, Policy500>
struct Policy600 : detail::chained_policy<600, Policy600, Policy500>
{
static constexpr int threads_per_block = 256;
static constexpr int items_per_thread = 16;
Expand Down Expand Up @@ -307,7 +307,7 @@ struct policy_hub
NoScaling<ReducePolicy::BLOCK_THREADS, ReducePolicy::ITEMS_PER_THREAD>>;
};

struct Policy1000 : ChainedPolicy<1000, Policy1000, Policy600>
struct Policy1000 : detail::chained_policy<1000, Policy1000, Policy600>
{
// Use values from tuning if a specialization exists, otherwise pick Policy600
template <typename Tuning>
Expand Down
10 changes: 5 additions & 5 deletions cub/cub/device/dispatch/tuning/tuning_reduce_by_key.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -920,7 +920,7 @@ struct policy_hub

struct Policy500
: DefaultPolicy500
, ChainedPolicy<500, Policy500, Policy500>
, detail::chained_policy<500, Policy500, Policy500>
{};

// Use values from tuning if a specialization exists, otherwise pick DefaultPolicy
Expand All @@ -936,7 +936,7 @@ struct policy_hub
template <typename Tuning>
static auto select_agent_policy(long) -> typename DefaultPolicy<LOAD_DEFAULT>::ReduceByKeyPolicyT;

struct Policy800 : ChainedPolicy<800, Policy800, Policy500>
struct Policy800 : detail::chained_policy<800, Policy800, Policy500>
{
using ReduceByKeyPolicyT =
decltype(select_agent_policy<sm80_tuning<KeyT, AccumT, is_primitive_op<ReductionOpT>()>>(0));
Expand All @@ -947,16 +947,16 @@ struct policy_hub

struct Policy860
: DefaultPolicy860
, ChainedPolicy<860, Policy860, Policy800>
, detail::chained_policy<860, Policy860, Policy800>
{};

struct Policy900 : ChainedPolicy<900, Policy900, Policy860>
struct Policy900 : detail::chained_policy<900, Policy900, Policy860>
{
using ReduceByKeyPolicyT =
decltype(select_agent_policy<sm90_tuning<KeyT, AccumT, is_primitive_op<ReductionOpT>()>>(0));
};

struct Policy1000 : ChainedPolicy<1000, Policy1000, Policy900>
struct Policy1000 : detail::chained_policy<1000, Policy1000, Policy900>
{
// Use values from tuning if a specialization exists, otherwise fall back to SM90
template <typename Tuning>
Expand Down
10 changes: 5 additions & 5 deletions cub/cub/device/dispatch/tuning/tuning_rle_encode.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -293,7 +293,7 @@ struct policy_hub

struct Policy500
: DefaultPolicy500
, ChainedPolicy<500, Policy500, Policy500>
, detail::chained_policy<500, Policy500, Policy500>
{};

// Use values from tuning if a specialization exists, otherwise pick the default
Expand All @@ -308,7 +308,7 @@ struct policy_hub
template <typename Tuning>
static auto select_agent_policy(long) -> typename DefaultPolicy<LOAD_DEFAULT>::ReduceByKeyPolicyT;

struct Policy800 : ChainedPolicy<800, Policy800, Policy500>
struct Policy800 : detail::chained_policy<800, Policy800, Policy500>
{
using ReduceByKeyPolicyT = decltype(select_agent_policy<sm80_tuning<LengthT, KeyT>>(0));
};
Expand All @@ -318,15 +318,15 @@ struct policy_hub

struct Policy860
: DefaultPolicy860
, ChainedPolicy<860, Policy860, Policy800>
, detail::chained_policy<860, Policy860, Policy800>
{};

struct Policy900 : ChainedPolicy<900, Policy900, Policy860>
struct Policy900 : detail::chained_policy<900, Policy900, Policy860>
{
using ReduceByKeyPolicyT = decltype(select_agent_policy<sm90_tuning<LengthT, KeyT>>(0));
};

struct Policy1000 : ChainedPolicy<1000, Policy1000, Policy900>
struct Policy1000 : detail::chained_policy<1000, Policy1000, Policy900>
{
// Use values from tuning if a specialization exists, otherwise pick Policy900
template <typename Tuning>
Expand Down
10 changes: 5 additions & 5 deletions cub/cub/device/dispatch/tuning/tuning_rle_non_trivial_runs.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,7 @@ struct policy_hub

struct Policy500
: DefaultPolicy500
, ChainedPolicy<500, Policy500, Policy500>
, detail::chained_policy<500, Policy500, Policy500>
{};

// Use values from tuning if a specialization exists, otherwise pick the default
Expand All @@ -330,7 +330,7 @@ struct policy_hub
static auto select_agent_policy(long) ->
typename DefaultPolicy<BLOCK_LOAD_WARP_TRANSPOSE, LengthT, LOAD_DEFAULT>::RleSweepPolicyT;

struct Policy800 : ChainedPolicy<800, Policy800, Policy500>
struct Policy800 : detail::chained_policy<800, Policy800, Policy500>
{
using RleSweepPolicyT = decltype(select_agent_policy<sm80_tuning<LengthT, KeyT>>(0));
};
Expand All @@ -341,15 +341,15 @@ struct policy_hub

struct Policy860
: DefaultPolicy860
, ChainedPolicy<860, Policy860, Policy800>
, detail::chained_policy<860, Policy860, Policy800>
{};

struct Policy900 : ChainedPolicy<900, Policy900, Policy860>
struct Policy900 : detail::chained_policy<900, Policy900, Policy860>
{
using RleSweepPolicyT = decltype(select_agent_policy<sm90_tuning<LengthT, KeyT>>(0));
};

struct Policy1000 : ChainedPolicy<1000, Policy1000, Policy900>
struct Policy1000 : detail::chained_policy<1000, Policy1000, Policy900>
{
// Use values from tuning if a specialization exists, otherwise pick Policy900
template <typename Tuning>
Expand Down
16 changes: 8 additions & 8 deletions cub/cub/device/dispatch/tuning/tuning_scan.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -590,13 +590,13 @@ struct policy_hub
static constexpr BlockStoreAlgorithm scan_transposed_store =
large_values ? BLOCK_STORE_WARP_TRANSPOSE_TIMESLICED : BLOCK_STORE_WARP_TRANSPOSE;

struct Policy500 : ChainedPolicy<500, Policy500, Policy500>
struct Policy500 : detail::chained_policy<500, Policy500, Policy500>
{
// GTX Titan: 29.5B items/s (232.4 GB/s) @ 48M 32-bit T
using ScanPolicyT =
agent_scan_policy<128, 12, AccumT, BLOCK_LOAD_DIRECT, LOAD_CA, BLOCK_STORE_WARP_TRANSPOSE_TIMESLICED, BLOCK_SCAN_RAKING>;
};
struct Policy520 : ChainedPolicy<520, Policy520, Policy500>
struct Policy520 : detail::chained_policy<520, Policy520, Policy500>
{
// Titan X: 32.47B items/s @ 48M 32-bit T
using ScanPolicyT =
Expand All @@ -611,7 +611,7 @@ struct policy_hub

struct Policy600
: DefaultPolicy
, ChainedPolicy<600, Policy600, Policy520>
, detail::chained_policy<600, Policy600, Policy520>
{};

// Use values from tuning if a specialization exists, otherwise pick DefaultPolicy
Expand All @@ -629,7 +629,7 @@ struct policy_hub
template <typename Tuning>
_CCCL_HOST_DEVICE static auto select_agent_policy(long) -> typename DefaultPolicy::ScanPolicyT;

struct Policy750 : ChainedPolicy<750, Policy750, Policy600>
struct Policy750 : detail::chained_policy<750, Policy750, Policy600>
{
// Use values from tuning if a specialization exists that matches a benchmark, otherwise pick Policy600
template <typename Tuning,
Expand Down Expand Up @@ -659,7 +659,7 @@ struct policy_hub
decltype(select_agent_policy750<sm75_tuning<InputValueT, AccumT, OffsetT, classify_op<ScanOpT>>, InputValueT>(0));
};

struct Policy800 : ChainedPolicy<800, Policy800, Policy750>
struct Policy800 : detail::chained_policy<800, Policy800, Policy750>
{
using ScanPolicyT =
decltype(select_agent_policy<sm80_tuning<classify_type<AccumT>,
Expand All @@ -670,15 +670,15 @@ struct policy_hub

struct Policy860
: DefaultPolicy
, ChainedPolicy<860, Policy860, Policy800>
, detail::chained_policy<860, Policy860, Policy800>
{};

struct Policy900 : ChainedPolicy<900, Policy900, Policy860>
struct Policy900 : detail::chained_policy<900, Policy900, Policy860>
{
using ScanPolicyT = decltype(select_agent_policy<sm90_tuning<AccumT, is_primitive_op<ScanOpT>()>>(0));
};

struct Policy1000 : ChainedPolicy<1000, Policy1000, Policy900>
struct Policy1000 : detail::chained_policy<1000, Policy1000, Policy900>
{
// Use values from tuning if a specialization exists that matches a benchmark, otherwise pick Policy900
template <typename Tuning,
Expand Down
12 changes: 6 additions & 6 deletions cub/cub/device/dispatch/tuning/tuning_scan_by_key.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -935,7 +935,7 @@ struct policy_hub
static constexpr int max_input_bytes = static_cast<int>((::cuda::std::max) (sizeof(key_t), sizeof(AccumT)));
static constexpr int combined_input_bytes = static_cast<int>(sizeof(key_t) + sizeof(AccumT));

struct Policy500 : ChainedPolicy<500, Policy500, Policy500>
struct Policy500 : detail::chained_policy<500, Policy500, Policy500>
{
static constexpr int nominal_4b_items_per_thread = 6;
static constexpr int items_per_thread =
Expand Down Expand Up @@ -973,7 +973,7 @@ struct policy_hub

struct Policy520
: DefaultPolicy520
, ChainedPolicy<520, Policy520, Policy500>
, detail::chained_policy<520, Policy520, Policy500>
{};

// Use values from tuning if a specialization exists, otherwise pick the default
Expand All @@ -991,7 +991,7 @@ struct policy_hub
// FIXME(bgruber): should we rather use `AccumT` instead of `ValueT` like the other default policies?
static auto select_agent_policy(long) -> typename DefaultPolicy<LOAD_DEFAULT, ValueT>::ScanByKeyPolicyT;

struct Policy800 : ChainedPolicy<800, Policy800, Policy520>
struct Policy800 : detail::chained_policy<800, Policy800, Policy520>
{
using ScanByKeyPolicyT = decltype(select_agent_policy<sm80_tuning<key_t, ValueT, is_primitive_op<ScanOpT>()>>(0));
};
Expand All @@ -1001,15 +1001,15 @@ struct policy_hub

struct Policy860
: DefaultPolicy860
, ChainedPolicy<860, Policy860, Policy800>
, detail::chained_policy<860, Policy860, Policy800>
{};

struct Policy900 : ChainedPolicy<900, Policy900, Policy860>
struct Policy900 : detail::chained_policy<900, Policy900, Policy860>
{
using ScanByKeyPolicyT = decltype(select_agent_policy<sm90_tuning<key_t, ValueT, is_primitive_op<ScanOpT>()>>(0));
};

struct Policy1000 : ChainedPolicy<1000, Policy1000, Policy900>
struct Policy1000 : detail::chained_policy<1000, Policy1000, Policy900>
{
// Use values from tuning if a specialization exists, otherwise pick Policy900
template <typename Tuning>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ struct policy_selector_from_types
template <typename AccumT, typename OffsetT, typename ReductionOpT>
struct policy_hub
{
struct Policy500 : ChainedPolicy<500, Policy500, Policy500>
struct Policy500 : detail::chained_policy<500, Policy500, Policy500>
{
private:
static constexpr int items_per_vec_load = 4;
Expand Down
Loading
Loading