BRL-CAD/oneTBB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Distributed size for concurrent ordered containers (#1803)

Browse Source 
Signed-off-by: Isaev, Ilya <ilya.isaev@intel.com>
Co-authored-by: Ilya Isaev <ilya.isaev@intel.com>
Co-authored-by: Aleksei Fedotov <aleksei.fedotov@intel.com>
This commit is contained in:
Konstantin Boyarinov
2026-02-24 11:47:06 +02:00
committed by GitHub
co-authored by Ilya Isaev Aleksei Fedotov
parent 50fe7ef4bf
commit f71cccc781
5 changed files with 151 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/oneapi/tbb/concurrent_map.h
+10 -9
View File
@@ -1,5 +1,6 @@
/*
Copyright (c) 2019-2021 Intel Corporation
Copyright (c) 2019-2025 Intel Corporation
Copyright (c) 2026 UXL Foundation Contributors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -26,7 +27,7 @@
namespace tbb {
namespace detail {
namespace d2 {
namespace d3 {
template<typename Key, typename Value, typename KeyCompare, typename RandomGenerator,
typename Allocator, bool AllowMultimapping>
@@ -68,8 +69,8 @@ template <typename Key, typename Value, typename Compare, typename Allocator>
class concurrent_multimap;
template <typename Key, typename Value, typename Compare = std::less<Key>, typename Allocator = tbb::tbb_allocator<std::pair<const Key, Value>>>
class concurrent_map : public concurrent_skip_list<map_traits<Key, Value, Compare, concurrent_geometric_level_generator<32>, Allocator, false>> {
using base_type = concurrent_skip_list<map_traits<Key, Value, Compare, concurrent_geometric_level_generator<32>, Allocator, false>>;
class concurrent_map : public concurrent_skip_list<map_traits<Key, Value, Compare, geometric_level_generator<32>, Allocator, false>> {
using base_type = concurrent_skip_list<map_traits<Key, Value, Compare, geometric_level_generator<32>, Allocator, false>>;
public:
using key_type = Key;
using mapped_type = Value;
@@ -218,8 +219,8 @@ void swap( concurrent_map<Key, Value, Compare, Allocator>& lhs,
}
template <typename Key, typename Value, typename Compare = std::less<Key>, typename Allocator = tbb::tbb_allocator<std::pair<const Key, Value>>>
class concurrent_multimap : public concurrent_skip_list<map_traits<Key, Value, Compare, concurrent_geometric_level_generator<32>, Allocator, true>> {
using base_type = concurrent_skip_list<map_traits<Key, Value, Compare, concurrent_geometric_level_generator<32>, Allocator, true>>;
class concurrent_multimap : public concurrent_skip_list<map_traits<Key, Value, Compare, geometric_level_generator<32>, Allocator, true>> {
using base_type = concurrent_skip_list<map_traits<Key, Value, Compare, geometric_level_generator<32>, Allocator, true>>;
public:
using key_type = Key;
using mapped_type = Value;
@@ -335,13 +336,13 @@ void swap( concurrent_multimap<Key, Value, Compare, Allocator>& lhs,
lhs.swap(rhs);
}
} // namespace d2
} // namespace d3
} // namespace detail
inline namespace v1 {
using detail::d2::concurrent_map;
using detail::d2::concurrent_multimap;
using detail::d3::concurrent_map;
using detail::d3::concurrent_multimap;
using detail::split;
} // inline namespace v1
include/oneapi/tbb/concurrent_set.h
+10 -9
View File
@@ -1,5 +1,6 @@
/*
Copyright (c) 2019-2021 Intel Corporation
Copyright (c) 2019-2025 Intel Corporation
Copyright (c) 2026 UXL Foundation Contributors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -25,7 +26,7 @@
namespace tbb {
namespace detail {
namespace d2 {
namespace d3 {
template<typename Key, typename KeyCompare, typename RandomGenerator, typename Allocator, bool AllowMultimapping>
struct set_traits {
@@ -52,8 +53,8 @@ template <typename Key, typename Compare, typename Allocator>
class concurrent_multiset;
template <typename Key, typename Compare = std::less<Key>, typename Allocator = tbb::tbb_allocator<Key>>
class concurrent_set : public concurrent_skip_list<set_traits<Key, Compare, concurrent_geometric_level_generator<32>, Allocator, false>> {
using base_type = concurrent_skip_list<set_traits<Key, Compare, concurrent_geometric_level_generator<32>, Allocator, false>>;
class concurrent_set : public concurrent_skip_list<set_traits<Key, Compare, geometric_level_generator<32>, Allocator, false>> {
using base_type = concurrent_skip_list<set_traits<Key, Compare, geometric_level_generator<32>, Allocator, false>>;
public:
using key_type = Key;
using value_type = typename base_type::value_type;
@@ -153,8 +154,8 @@ void swap( concurrent_set<Key, Compare, Allocator>& lhs,
}
template <typename Key, typename Compare = std::less<Key>, typename Allocator = tbb::tbb_allocator<Key>>
class concurrent_multiset : public concurrent_skip_list<set_traits<Key, Compare, concurrent_geometric_level_generator<32>, Allocator, true>> {
using base_type = concurrent_skip_list<set_traits<Key, Compare, concurrent_geometric_level_generator<32>, Allocator, true>>;
class concurrent_multiset : public concurrent_skip_list<set_traits<Key, Compare, geometric_level_generator<32>, Allocator, true>> {
using base_type = concurrent_skip_list<set_traits<Key, Compare, geometric_level_generator<32>, Allocator, true>>;
public:
using key_type = Key;
using value_type = typename base_type::value_type;
@@ -252,13 +253,13 @@ void swap( concurrent_multiset<Key, Compare, Allocator>& lhs,
lhs.swap(rhs);
}
} // namespace d2
} // namespace d3
} // namespace detail
inline namespace v1 {
using detail::d2::concurrent_set;
using detail::d2::concurrent_multiset;
using detail::d3::concurrent_set;
using detail::d3::concurrent_multiset;
using detail::split;
} // inline namespace v1
include/oneapi/tbb/detail/_concurrent_skip_list.h
+98 -41
View File
@@ -1,5 +1,6 @@
/*
Copyright (c) 2019-2025 Intel Corporation
Copyright (c) 2026 UXL Foundation Contributors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -49,7 +50,31 @@
namespace tbb {
namespace detail {
namespace d2 {
namespace d3 {
template <typename LevelGenerator, typename SizeType>
class skip_list_thread_data {
public:
using level_generator_type = LevelGenerator;
using size_type = SizeType;
skip_list_thread_data() : my_local_size(0) {}
std::size_t random_level() { return my_rng(); }
std::size_t local_size() const { return my_local_size.load(std::memory_order_relaxed); }
void set_size(size_type size) { my_local_size.store(size, std::memory_order_relaxed); }
void increment_size() {
// Not using my_local_size.fetch_add(1) to avoid exclusive locking of the cache line
// Since only one thread can write to this atomic, and no synchronization needed with readers
my_local_size.store(local_size() + 1, std::memory_order_relaxed);
}
private:
level_generator_type my_rng;
// Atomic to avoid formal data race between several readers and single writer
std::atomic<size_type> my_local_size;
};
template <typename Value, typename Allocator>
class skip_list_node {
@@ -276,6 +301,9 @@ protected:
using const_pointer = typename allocator_traits_type::const_pointer;
using random_level_generator_type = typename container_traits::random_level_generator_type;
using thread_data_type = skip_list_thread_data<random_level_generator_type, size_type>;
using ets_type = tbb::enumerable_thread_specific<thread_data_type, cache_aligned_allocator<thread_data_type>,
ets_suspend_aware>;
using node_ptr = list_node_type*;
@@ -286,10 +314,13 @@ private:
public:
static constexpr bool allow_multimapping = container_traits::allow_multimapping;
concurrent_skip_list() : my_head_ptr(nullptr), my_size(0), my_max_height(0) {}
concurrent_skip_list()
: my_head_ptr(nullptr)
, my_max_height(0) {}
explicit concurrent_skip_list( const key_compare& comp, const allocator_type& alloc = allocator_type() )
: my_node_allocator(alloc), my_compare(comp), my_head_ptr(nullptr), my_size(0), my_max_height(0) {}
: my_node_allocator(alloc), my_compare(comp), my_head_ptr(nullptr)
, my_max_height(0) {}
explicit concurrent_skip_list( const allocator_type& alloc )
: concurrent_skip_list(key_compare(), alloc) {}
@@ -314,32 +345,36 @@ public:
: concurrent_skip_list(init, key_compare(), alloc) {}
concurrent_skip_list( const concurrent_skip_list& other )
: my_node_allocator(node_allocator_traits::select_on_container_copy_construction(other.get_allocator())),
my_compare(other.my_compare), my_rng(other.my_rng), my_head_ptr(nullptr),
my_size(0), my_max_height(0)
: my_node_allocator(node_allocator_traits::select_on_container_copy_construction(other.get_allocator()))
, my_compare(other.my_compare)
, my_head_ptr(nullptr)
, my_max_height(0)
{
internal_copy(other);
__TBB_ASSERT(my_size == other.my_size, "Wrong size of copy-constructed container");
__TBB_ASSERT(size() == other.size(), "Wrong size of copy-constructed container");
}
concurrent_skip_list( const concurrent_skip_list& other, const allocator_type& alloc )
: my_node_allocator(alloc), my_compare(other.my_compare), my_rng(other.my_rng), my_head_ptr(nullptr),
my_size(0), my_max_height(0)
: my_node_allocator(alloc), my_compare(other.my_compare)
, my_head_ptr(nullptr)
, my_max_height(0)
{
internal_copy(other);
__TBB_ASSERT(my_size == other.my_size, "Wrong size of copy-constructed container");
__TBB_ASSERT(size() == other.size(), "Wrong size of copy-constructed container");
}
concurrent_skip_list( concurrent_skip_list&& other )
: my_node_allocator(std::move(other.my_node_allocator)), my_compare(other.my_compare),
my_rng(std::move(other.my_rng)), my_head_ptr(nullptr) // my_head_ptr would be stored in internal_move
: my_node_allocator(std::move(other.my_node_allocator))
, my_compare(other.my_compare)
, my_head_ptr(nullptr) // my_head_ptr would be stored in internal_move
{
internal_move(std::move(other));
}
concurrent_skip_list( concurrent_skip_list&& other, const allocator_type& alloc )
: my_node_allocator(alloc), my_compare(other.my_compare),
my_rng(std::move(other.my_rng)), my_head_ptr(nullptr)
: my_node_allocator(alloc)
, my_compare(other.my_compare)
, my_head_ptr(nullptr)
{
using is_always_equal = typename allocator_traits_type::is_always_equal;
internal_move_construct_with_allocator(std::move(other), is_always_equal());
@@ -355,7 +390,6 @@ public:
clear();
copy_assign_allocators(my_node_allocator, other.my_node_allocator);
my_compare = other.my_compare;
my_rng = other.my_rng;
internal_copy(other);
}
return *this;
@@ -367,8 +401,6 @@ public:
delete_head();
my_compare = std::move(other.my_compare);
my_rng = std::move(other.my_rng);
move_assign_allocators(my_node_allocator, other.my_node_allocator);
using pocma_type = typename node_allocator_traits::propagate_on_container_move_assignment;
using is_always_equal = typename node_allocator_traits::is_always_equal;
@@ -417,7 +449,8 @@ public:
std::pair<iterator, bool> insert( node_type&& nh ) {
if (!nh.empty()) {
auto insert_node = d1::node_handle_accessor::get_node_ptr(nh);
std::pair<iterator, bool> insert_result = internal_insert_node(insert_node);
thread_data_type& td = my_ets.local();
std::pair<iterator, bool> insert_result = internal_insert_node(insert_node, td);
if (insert_result.second) {
d1::node_handle_accessor::deactivate(nh);
}
@@ -589,7 +622,7 @@ public:
head->set_next(level, nullptr);
}
my_size.store(0, std::memory_order_relaxed);
my_ets.clear();
my_max_height.store(0, std::memory_order_relaxed);
}
@@ -618,7 +651,13 @@ public:
}
size_type size() const {
return my_size.load(std::memory_order_relaxed);
size_type total_size = 0;
my_ets.combine_each([&](const thread_data_type& td) {
total_size += td.local_size();
});
return total_size;
}
size_type max_size() const {
@@ -744,8 +783,11 @@ private:
my_head_ptr.store(other.my_head_ptr.load(std::memory_order_relaxed), std::memory_order_relaxed);
other.my_head_ptr.store(nullptr, std::memory_order_relaxed);
my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed);
other.my_size.store(0, std::memory_order_relaxed);
// TODO: investigate whether the performance gain from shrinking sizes to calling thread's size is justified
// Logic similar to enumerable_thread_specific::internal_move can be used instead
my_ets.clear();
my_ets.local().set_size(other.size());
other.my_ets.clear();
my_max_height.store(other.my_max_height.load(std::memory_order_relaxed), std::memory_order_relaxed);
other.my_max_height.store(0, std::memory_order_relaxed);
@@ -761,7 +803,9 @@ private:
if (my_node_allocator == other.get_allocator()) {
internal_move(std::move(other));
} else {
my_size.store(0, std::memory_order_relaxed);
my_ets.clear();
my_ets.local().set_size(0);
my_max_height.store(other.my_max_height.load(std::memory_order_relaxed), std::memory_order_relaxed);
internal_copy(std::make_move_iterator(other.begin()), std::make_move_iterator(other.end()));
}
@@ -941,15 +985,17 @@ private:
template<typename... Args>
std::pair<iterator, bool> internal_insert( Args&&... args ) {
node_ptr new_node = create_value_node(std::forward<Args>(args)...);
std::pair<iterator, bool> insert_result = internal_insert_node(new_node);
thread_data_type& td = my_ets.local();
node_ptr new_node = create_value_node(td, std::forward<Args>(args)...);
std::pair<iterator, bool> insert_result = internal_insert_node(new_node, td);
if (!insert_result.second) {
delete_value_node(new_node);
}
return insert_result;
}
std::pair<iterator, bool> internal_insert_node( node_ptr new_node ) {
std::pair<iterator, bool> internal_insert_node( node_ptr new_node, thread_data_type& td )
{
array_type prev_nodes;
array_type curr_nodes;
size_type new_height = new_node->height();
@@ -1006,7 +1052,7 @@ private:
}
}
}
++my_size;
td.increment_size();
return std::pair<iterator, bool>(iterator(new_node), true);
}
}
@@ -1047,7 +1093,12 @@ private:
prev_nodes[level]->set_next(level, erase_node->next(level));
erase_node->set_next(level, nullptr);
}
my_size.fetch_sub(1, std::memory_order_relaxed);
// Shrink all thread-local sizes into the calling thread's size and decrement
// TODO: investigate if performance gain of this optimization is justified
size_type total_size = size();
my_ets.clear();
my_ets.local().set_size(total_size - 1);
result.first = erase_node;
result.second = next_node;
@@ -1100,8 +1151,9 @@ private:
}
template <typename... Args>
node_ptr create_value_node( Args&&... args ) {
node_ptr node = create_node(my_rng());
node_ptr create_value_node( thread_data_type& td, Args&&... args )
{
node_ptr node = create_node(td.random_level());
// try_call API is not convenient here due to broken
// variadic capture on GCC 4.8.5
@@ -1179,11 +1231,16 @@ private:
using std::swap;
swap_allocators(my_node_allocator, other.my_node_allocator);
swap(my_compare, other.my_compare);
swap(my_rng, other.my_rng);
swap_atomics_relaxed(my_head_ptr, other.my_head_ptr);
swap_atomics_relaxed(my_size, other.my_size);
swap_atomics_relaxed(my_max_height, other.my_max_height);
// TODO: investigate whether the performance gain from shrinking sizes to calling thread's size is justified
// Logic similar to enumerable_thread_specific::internal_swap can be used instead
size_type total_size = size();
my_ets.clear();
my_ets.local().set_size(other.size());
other.my_ets.clear();
other.my_ets.local().set_size(total_size);
}
void internal_swap( concurrent_skip_list& other, /*POCMA || is_always_equal =*/std::true_type ) {
@@ -1197,10 +1254,9 @@ private:
node_allocator_type my_node_allocator;
key_compare my_compare;
random_level_generator_type my_rng;
atomic_node_ptr my_head_ptr;
std::atomic<size_type> my_size;
std::atomic<size_type> my_max_height;
mutable ets_type my_ets; // mutable is needed since ets::combine_each is non-const
template<typename OtherTraits>
friend class concurrent_skip_list;
@@ -1259,27 +1315,28 @@ bool operator>=( const concurrent_skip_list<Traits>& lhs, const concurrent_skip_
// Generates a number from the interval [0, MaxLevel).
template <std::size_t MaxLevel>
class concurrent_geometric_level_generator {
class geometric_level_generator {
public:
static constexpr std::size_t max_level = MaxLevel;
// TODO: modify the algorithm to accept other values of max_level
static_assert(max_level == 32, "Incompatible max_level for rng");
concurrent_geometric_level_generator() : engines(std::minstd_rand::result_type(time(nullptr))) {}
geometric_level_generator() : my_engine(std::minstd_rand::result_type(time(nullptr))) {}
std::size_t operator()() {
// +1 is required to pass at least 1 into log2 (log2(0) is undefined)
// -1 is required to have an ability to return 0 from the generator (max_level - log2(2^31) - 1)
std::size_t result = max_level - std::size_t(tbb::detail::log2(engines.local()() + 1)) - 1;
auto random_number = my_engine();
std::size_t result = max_level - std::size_t(tbb::detail::log2(random_number + 1)) - 1;
__TBB_ASSERT(result <= max_level, nullptr);
return result;
}
private:
tbb::enumerable_thread_specific<std::minstd_rand> engines;
std::minstd_rand my_engine;
};
} // namespace d2
} // namespace d3
} // namespace detail
} // namespace tbb
integration/windows/vs/onetbb.natvis
+32 -13
View File
@@ -1,6 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
Copyright (c) 2025 Intel Corporation
Copyright (c) 2026 UXL Foundation Contributors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -82,25 +83,43 @@
</Expand>
</Type>
<Type Name="tbb::detail::d2::concurrent_map&lt;*,*,*,*&gt;">
<AlternativeType Name="tbb::detail::d2::concurrent_multimap&lt;*,*,*,*&gt;"/>
<AlternativeType Name="tbb::detail::d2::concurrent_set&lt;*,*,*&gt;"/>
<AlternativeType Name="tbb::detail::d2::concurrent_multiset&lt;*,*,*&gt;"/>
<DisplayString>{{ size = {my_size} }}</DisplayString>
<Type Name="tbb::detail::d3::concurrent_map&lt;*,*,*,*&gt;">
<AlternativeType Name="tbb::detail::d3::concurrent_multimap&lt;*,*,*,*&gt;"/>
<AlternativeType Name="tbb::detail::d3::concurrent_set&lt;*,*,*&gt;"/>
<AlternativeType Name="tbb::detail::d3::concurrent_multiset&lt;*,*,*&gt;"/>
<Expand>
<CustomListItems MaxItemsPerView="1">
<!--
The size of ordered associative containers is actually located in thread local storage.
So, in order to display the size of the container, we have to traverse and sum all elements of TLS.
Custom intrinsics do not support loops, only simple expressions, so [size] can not
be displayed in DisplayString section.
-->
<Variable Name="size" InitialValue="0"/>
<Variable Name="index" InitialValue="0"/>
<Variable Name="segment_index" InitialValue="-1"/>
<Loop>
<Break Condition="index == my_ets.my_locals.my_size._Storage._Value"/>
<Exec>segment_index = __log2(1 | index)</Exec>
<Exec>size += (*(thread_data_type*)(&amp;my_ets.my_locals.my_segment_table._Storage._Value[segment_index]._Storage._Value[index])).my_local_size._Storage._Value</Exec>
<Exec>index += 1</Exec>
</Loop>
<Item Name="[size]">size</Item>
</CustomListItems>
<Item Name="[comparator]">my_compare</Item>
<Item Name="[allocator]">my_node_allocator</Item>
<CustomListItems MaxItemsPerView="50">
<!--
concurrent ordered containers implement a concurrent skip list
where each node has multiple pointers to next nodes on each level
only the first level sub-list contains all nodes in the list
pointers to the next nodes are not stored in the node class directly
but are located in the memory following the node
next pointer on the first level can be obtained as (std::atomic<list_node_type*>)(node_address + 1)
the first node in the list is always a dummy node so the first real item is located in the second node
Concurrent ordered containers implement a concurrent skip list
where each node has multiple pointers to next nodes on each level.
Only the first level sub-list contains all nodes in the list.
Pointers to the next nodes are not stored in the node class directly
but are located in the memory following the node.
Next pointer on the first level can be obtained as (std::atomic<list_node_type*>)(node_address + 1)
The first node in the list is always a dummy node so the first real item is located in the second node.
-->
<Variable Name="node_ptr" InitialValue="(*(atomic_node_ptr*)(my_head_ptr._Storage._Value + 1))._Storage._Value"/>
test/common/config.h
+1 -1
View File
@@ -1,6 +1,6 @@
/*
Copyright (c) 2005-2025 Intel Corporation
Copyright (c) 2025 UXL Foundation Сontributors
Copyright (c) 2026 UXL Foundation Contributors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.