From 2f75565c140c73f658a93bafa0a5b694a90bca14 Mon Sep 17 00:00:00 2001 From: Danila Klimenko Date: Tue, 12 May 2015 20:33:00 +0200 Subject: [PATCH] chromatic_tree: Fine-grained locking implementation (complete with rebalancing) --- containers_cpp/include/embb/containers/internal/hazard_pointer-inl.h | 10 +++++++++- containers_cpp/include/embb/containers/internal/hazard_pointer.h | 9 ++++++++- containers_cpp/include/embb/containers/internal/lock_free_chromatic_tree-inl.h | 511 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- containers_cpp/include/embb/containers/internal/lock_free_chromatic_tree-rebalance.h | 712 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ containers_cpp/include/embb/containers/lock_free_chromatic_tree.h | 123 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------------- containers_cpp/test/tree_test-inl.h | 10 +++++----- 6 files changed, 812 insertions(+), 563 deletions(-) diff --git a/containers_cpp/include/embb/containers/internal/hazard_pointer-inl.h b/containers_cpp/include/embb/containers/internal/hazard_pointer-inl.h index 3873dba..5f62926 100644 --- a/containers_cpp/include/embb/containers/internal/hazard_pointer-inl.h +++ b/containers_cpp/include/embb/containers/internal/hazard_pointer-inl.h @@ -182,7 +182,9 @@ HazardPointerThreadEntry::~HazardPointerThreadEntry() { } template< typename GuardType > -GuardType HazardPointerThreadEntry::GetGuard(int pos) const { +typename HazardPointerThreadEntry::AtomicGuard& +HazardPointerThreadEntry::GetGuard(int pos) const { + assert(pos >= 0 && pos < guards_per_thread); return guarded_pointers[pos]; } @@ -393,6 +395,12 @@ void HazardPointer< GuardType >::GuardPointer(int guardPosition, } template< typename GuardType > +typename HazardPointer< GuardType >::AtomicGuard& +HazardPointer< GuardType >::GetGuardedPointer(int guardPosition) { + return GetHazardPointerElementForCurrentThread().GetGuard(guardPosition); +} + +template< typename GuardType > void HazardPointer< GuardType >::EnqueuePointerForDeletion( GuardType guardedElement) { GetHazardPointerElementForCurrentThread().AddRetired(guardedElement); diff --git a/containers_cpp/include/embb/containers/internal/hazard_pointer.h b/containers_cpp/include/embb/containers/internal/hazard_pointer.h index e1e5f49..7387bf1 100644 --- a/containers_cpp/include/embb/containers/internal/hazard_pointer.h +++ b/containers_cpp/include/embb/containers/internal/hazard_pointer.h @@ -238,6 +238,8 @@ class HazardPointerThreadEntry { HazardPointerThreadEntry & operator= (const HazardPointerThreadEntry&); public: + typedef embb::base::Atomic< GuardType > AtomicGuard; + /** * Checks if current thread is active (with respect to participating in hazard * pointer management) @@ -318,7 +320,7 @@ class HazardPointerThreadEntry { * Gets the guard at the specified position. * Positions are numbered, beginning with 0. */ - GuardType GetGuard( + AtomicGuard& GetGuard( int pos /**< [IN] Position of the guard */) const; @@ -467,6 +469,8 @@ class HazardPointer { can be deleted*/); public: + typedef typename HazardPointerThreadEntry_t::AtomicGuard AtomicGuard; + /** * Gets the capacity of one retired list * @@ -516,6 +520,9 @@ class HazardPointer { * Guards \c guardedElement with the guard at position \c guardPosition */ void GuardPointer(int guardPosition, GuardType guardedElement); + + AtomicGuard& GetGuardedPointer(int guardPosition); + /** * Enqueue a pointer for deletion. It is added to the retired list and * deleted when no thread accesses it anymore. diff --git a/containers_cpp/include/embb/containers/internal/lock_free_chromatic_tree-inl.h b/containers_cpp/include/embb/containers/internal/lock_free_chromatic_tree-inl.h index 9f6e214..d33d5a3 100644 --- a/containers_cpp/include/embb/containers/internal/lock_free_chromatic_tree-inl.h +++ b/containers_cpp/include/embb/containers/internal/lock_free_chromatic_tree-inl.h @@ -105,7 +105,7 @@ ChromaticTree(size_t capacity, Key undefined_key, Value undefined_value, #ifdef EMBB_PLATFORM_COMPILER_MSVC #pragma warning(pop) #endif - node_hazard_manager_(free_node_callback_, NULL, 8), + node_hazard_manager_(free_node_callback_, NULL, 10), undefined_key_(undefined_key), undefined_value_(undefined_value), compare_(compare), @@ -128,8 +128,9 @@ ChromaticTree:: template bool ChromaticTree:: Get(const Key& key, Value& value) { - NodePtr leaf; - Search(key, leaf); + HazardNodePtr parent(node_hazard_manager_.GetGuardedPointer(0), NULL); + HazardNodePtr leaf (node_hazard_manager_.GetGuardedPointer(1), NULL); + Search(key, leaf, parent); bool keys_are_equal = !IsSentinel(leaf) && !(compare_(key, leaf->GetKey()) || @@ -139,8 +140,6 @@ Get(const Key& key, Value& value) { value = leaf->GetValue(); } - node_hazard_manager_.GuardPointer(0, NULL); - return keys_are_equal; } @@ -158,9 +157,11 @@ TryInsert(const Key& key, const Value& value, Value& old_value) { NodePtr new_sibling = NULL; NodePtr new_parent = NULL; bool insertion_succeeded = false; + bool added_violation = false; while (!insertion_succeeded) { - NodePtr leaf, parent; + HazardNodePtr parent(node_hazard_manager_.GetGuardedPointer(0), NULL); + HazardNodePtr leaf (node_hazard_manager_.GetGuardedPointer(1), NULL); Search(key, leaf, parent); // Try to lock the parent @@ -200,23 +201,21 @@ TryInsert(const Key& key, const Value& value, Value& old_value) { if (new_parent == NULL) break; } - GetPointerToChild(parent, leaf).CompareAndSwap(leaf, new_parent); - - insertion_succeeded = true; + NodePtr expected = leaf; + insertion_succeeded = GetPointerToChild(parent, leaf) + .CompareAndSwap(expected, new_parent); + assert(insertion_succeeded); // For now (FGL tree) this CAS may not fail + if (!insertion_succeeded) continue; - leaf->Retire(); - leaf_lock.Unlock(); - node_hazard_manager_.GuardPointer(1, NULL); - RemoveNode(leaf); + RetireHazardousNode(leaf, leaf_lock); -// if (parent->GetWeight() == 0 && new_parent->GetWeight() == 0) { -// CleanUp(key); -// } + added_violation = (parent->GetWeight() == 0 && + new_parent->GetWeight() == 0); } - node_hazard_manager_.GuardPointer(0, NULL); - node_hazard_manager_.GuardPointer(1, NULL); - if (!insertion_succeeded) { + if (insertion_succeeded) { + if (added_violation) CleanUp(key); + } else { if (new_leaf != NULL) FreeNode(new_leaf); if (new_sibling != NULL) FreeNode(new_sibling); if (new_parent != NULL) FreeNode(new_parent); @@ -237,9 +236,12 @@ bool ChromaticTree:: TryDelete(const Key& key, Value& old_value) { NodePtr new_leaf = NULL; bool deletion_succeeded = false; + bool added_violation = false; while (!deletion_succeeded) { - NodePtr leaf, parent, grandparent; + HazardNodePtr grandparent(node_hazard_manager_.GetGuardedPointer(0), NULL); + HazardNodePtr parent (node_hazard_manager_.GetGuardedPointer(1), NULL); + HazardNodePtr leaf (node_hazard_manager_.GetGuardedPointer(2), NULL); Search(key, leaf, parent, grandparent); // Reached leaf has a different key - nothing to delete @@ -259,16 +261,17 @@ TryDelete(const Key& key, Value& old_value) { // Try to lock the parent UniqueLock parent_lock(parent->GetMutex(), embb::base::try_lock); if (!parent_lock.OwnsLock() || parent->IsRetired()) continue; + + // Get the sibling (and protect it with hazard pointer) + HazardNodePtr sibling(node_hazard_manager_.GetGuardedPointer(3), NULL); + sibling.ProtectHazard((parent->GetLeft() == leaf) ? + parent->GetRight() : parent->GetLeft()); + if (parent->IsRetired() || !sibling.IsActive()) continue; + VERIFY_ADDRESS(static_cast(sibling)); + // Verify that the leaf is still the parent's child if (!HasChild(parent, leaf)) continue; - AtomicNodePtr& other_child = ((parent->GetLeft() == leaf) ? - parent->GetRight() : parent->GetLeft()); - NodePtr sibling = other_child; - node_hazard_manager_.GuardPointer(3, sibling); - if (sibling != other_child) continue; - VERIFY_ADDRESS(sibling); - // Try to lock the sibling UniqueLock sibling_lock(sibling->GetMutex(), embb::base::try_lock); if (!sibling_lock.OwnsLock() || sibling->IsRetired()) continue; @@ -287,33 +290,22 @@ TryDelete(const Key& key, Value& old_value) { old_value = leaf->GetValue(); - GetPointerToChild(grandparent, parent).CompareAndSwap(parent, new_leaf); - - deletion_succeeded = true; - - parent->Retire(); - parent_lock.Unlock(); - node_hazard_manager_.GuardPointer(1, NULL); - RemoveNode(parent); - leaf->Retire(); - leaf_lock.Unlock(); - node_hazard_manager_.GuardPointer(2, NULL); - RemoveNode(leaf); - sibling->Retire(); - sibling_lock.Unlock(); - node_hazard_manager_.GuardPointer(3, NULL); - RemoveNode(sibling); - -// if (new_weight > 1) { -// CleanUp(key); -// } + NodePtr expected = parent; + deletion_succeeded = GetPointerToChild(grandparent, parent) + .CompareAndSwap(expected, new_leaf); + assert(deletion_succeeded); // For now (FGL tree) this CAS may not fail + if (!deletion_succeeded) continue; + + RetireHazardousNode(parent, parent_lock); + RetireHazardousNode(leaf, leaf_lock); + RetireHazardousNode(sibling, sibling_lock); + + added_violation = (new_weight > 1); } - node_hazard_manager_.GuardPointer(0, NULL); - node_hazard_manager_.GuardPointer(1, NULL); - node_hazard_manager_.GuardPointer(2, NULL); - node_hazard_manager_.GuardPointer(3, NULL); - if (!deletion_succeeded) { + if (deletion_succeeded) { + if (added_violation) CleanUp(key); + } else { if (new_leaf != NULL) FreeNode(new_leaf); } @@ -340,39 +332,22 @@ IsEmpty() { template void ChromaticTree:: -Search(const Key& key, NodePtr& leaf) { - NodePtr parent; - Search(key, leaf, parent); - node_hazard_manager_.GuardPointer(0, leaf); - node_hazard_manager_.GuardPointer(1, NULL); -} - -template -void ChromaticTree:: -Search(const Key& key, NodePtr& leaf, NodePtr& parent) { +Search(const Key& key, HazardNodePtr& leaf, HazardNodePtr& parent) { bool reached_leaf = false; while (!reached_leaf) { - parent = entry_; - node_hazard_manager_.GuardPointer(0, parent); - - leaf = entry_->GetLeft(); - node_hazard_manager_.GuardPointer(1, leaf); - if (leaf != entry_->GetLeft()) continue; + parent.ProtectHazard(entry_); + leaf.ProtectHazard(entry_->GetLeft()); + if (parent->IsRetired() || !leaf.IsActive()) continue; reached_leaf = IsLeaf(leaf); while (!reached_leaf) { - parent = leaf; - node_hazard_manager_.GuardPointer(0, parent); + parent.AdoptGuard(leaf); + leaf.ProtectHazard((IsSentinel(leaf) || compare_(key, leaf->GetKey())) ? + leaf->GetLeft() : leaf->GetRight()); + if (parent->IsRetired() || !leaf.IsActive()) break; + VERIFY_ADDRESS(static_cast(leaf)); - AtomicNodePtr& next_leaf = - (IsSentinel(leaf) || compare_(key, leaf->GetKey())) ? - leaf->GetLeft() : leaf->GetRight(); - leaf = next_leaf; - node_hazard_manager_.GuardPointer(1, leaf); - if (leaf != next_leaf || parent->IsRetired()) break; - - VERIFY_ADDRESS(leaf); reached_leaf = IsLeaf(leaf); } } @@ -380,35 +355,25 @@ Search(const Key& key, NodePtr& leaf, NodePtr& parent) { template void ChromaticTree:: -Search(const Key& key, NodePtr& leaf, NodePtr& parent, NodePtr& grandparent) { +Search(const Key& key, HazardNodePtr& leaf, HazardNodePtr& parent, + HazardNodePtr& grandparent) { bool reached_leaf = false; while (!reached_leaf) { - grandparent = NULL; - - parent = entry_; - node_hazard_manager_.GuardPointer(1, parent); - - leaf = entry_->GetLeft(); - node_hazard_manager_.GuardPointer(2, leaf); - if (leaf != entry_->GetLeft()) continue; + grandparent.ProtectHazard(entry_); + parent.ProtectHazard(entry_); + leaf.ProtectHazard(entry_->GetLeft()); + if (parent->IsRetired() || !leaf.IsActive()) continue; reached_leaf = IsLeaf(leaf); while (!reached_leaf) { - grandparent = parent; - node_hazard_manager_.GuardPointer(0, grandparent); - - parent = leaf; - node_hazard_manager_.GuardPointer(1, parent); - - AtomicNodePtr& next_leaf = - (IsSentinel(leaf) || compare_(key, leaf->GetKey())) ? - leaf->GetLeft() : leaf->GetRight(); - leaf = next_leaf; - node_hazard_manager_.GuardPointer(2, leaf); - if (leaf != next_leaf || parent->IsRetired()) break; + grandparent.AdoptGuard(parent); + parent.AdoptGuard(leaf); + leaf.ProtectHazard((IsSentinel(leaf) || compare_(key, leaf->GetKey())) ? + leaf->GetLeft() : leaf->GetRight()); + if (parent->IsRetired() || !leaf.IsActive()) break; + VERIFY_ADDRESS(static_cast(leaf)); - VERIFY_ADDRESS(leaf); reached_leaf = IsLeaf(leaf); } } @@ -423,13 +388,15 @@ IsLeaf(const NodePtr& node) const { template bool ChromaticTree:: IsSentinel(const NodePtr& node) const { - return (node == entry_) || (node == entry_->GetLeft()); + NodePtr entry = entry_; //Bug: "operator->()" is not const in AtomicPointer<> + return (node == entry) || (node == entry->GetLeft()); } template bool ChromaticTree:: HasFixedWeight(const NodePtr& node) const { - return (IsSentinel(node)) || (node == entry_->GetLeft()->GetLeft()); + NodePtr entry = entry_; //Bug: "operator->()" is not const in AtomicPointer<> + return (IsSentinel(node)) || (node == entry->GetLeft()->GetLeft()); } template @@ -470,7 +437,8 @@ GetHeight(const NodePtr& node) const { template bool ChromaticTree:: IsBalanced() const { - return IsBalanced(entry_->GetLeft()); + NodePtr entry = entry_; //Bug: "operator->()" is not const in AtomicPointer<> + return IsBalanced(entry->GetLeft()); } template @@ -509,92 +477,137 @@ FreeNode(NodePtr node) { template bool ChromaticTree:: CleanUp(const Key& key) { - for (;;) { - NodePtr grandgrandparent = NULL; - NodePtr grandparent = NULL; - NodePtr parent = entry_; - NodePtr leaf = entry_->GetLeft(); - - while (!IsLeaf(leaf) && (leaf->GetWeight() <= 1) && - (leaf->GetWeight() != 0 || parent->GetWeight() != 0)) { - grandgrandparent = grandparent; - grandparent = parent; - parent = leaf; - leaf = (IsSentinel(leaf) || compare_(key, leaf->GetKey())) ? - leaf->GetLeft() : leaf->GetRight(); - VERIFY_ADDRESS(leaf); - } + HazardNodePtr grangranparent(node_hazard_manager_.GetGuardedPointer(0), NULL); + HazardNodePtr grandparent (node_hazard_manager_.GetGuardedPointer(1), NULL); + HazardNodePtr parent (node_hazard_manager_.GetGuardedPointer(2), NULL); + HazardNodePtr leaf (node_hazard_manager_.GetGuardedPointer(3), NULL); + bool reached_leaf = false; - if (leaf->GetWeight() == 1) { - break; + while (!reached_leaf) { + bool found_violation = false; + + grangranparent.ProtectHazard(entry_); + grandparent.ProtectHazard(entry_); + parent.ProtectHazard(entry_); + leaf.ProtectHazard(entry_->GetLeft()); + if (parent->IsRetired() || !leaf.IsActive()) continue; + + reached_leaf = IsLeaf(leaf); + while (!reached_leaf && !found_violation) { + grangranparent.AdoptGuard(grandparent); + grandparent.AdoptGuard(parent); + parent.AdoptGuard(leaf); + leaf.ProtectHazard((IsSentinel(leaf) || compare_(key, leaf->GetKey())) ? + leaf->GetLeft() : leaf->GetRight()); + if (parent->IsRetired() || !leaf.IsActive()) break; + VERIFY_ADDRESS(static_cast(leaf)); + + found_violation = (leaf->GetWeight() > 1) || + (leaf->GetWeight() == 0 && parent->GetWeight() == 0); + + reached_leaf = IsLeaf(leaf); } - if (!Rebalance(grandgrandparent, grandparent, parent, leaf)) { - return false; + if (found_violation) { + reached_leaf = false; + + if (Rebalance(grangranparent, grandparent, parent, leaf) == EMBB_NOMEM) { + assert(false && "No memory for rebalancing!"); + return false; + } } } return true; } +#define PROTECT_NODE_WITH_LOCK(node, lock_name) \ + UniqueLock lock_name(node->GetMutex(), embb::base::try_lock); \ + if (!lock_name.OwnsLock() || node->IsRetired()) return EMBB_BUSY; + +#define DEFINE_NODE_WITH_HAZARD(h_num, node, parent, method) \ + HazardNodePtr node(node_hazard_manager_.GetGuardedPointer(h_num), NULL); \ + node.ProtectHazard(parent->method()); \ + if (parent->IsRetired() || !node.IsActive()) return EMBB_BUSY; \ + VERIFY_ADDRESS(static_cast(node)) + template -bool ChromaticTree:: -Rebalance(const NodePtr& u, const NodePtr& ux, const NodePtr& uxx, - const NodePtr& uxxx) { - //TODO: weakLLX(u); - if (!HasChild(u, ux)) return false; - - //TODO: weakLLX(ux); - NodePtr uxl = ux->GetLeft(); VERIFY_ADDRESS(uxl); - NodePtr uxr = ux->GetRight(); VERIFY_ADDRESS(uxr); - bool uxx_is_left = (uxx == uxl); - if (!HasChild(ux, uxx)) return false; - - //TODO: weakLLX(uxx); - NodePtr uxxl = uxx->GetLeft(); VERIFY_ADDRESS(uxxl); - NodePtr uxxr = uxx->GetRight(); VERIFY_ADDRESS(uxxr); +embb_errors_t ChromaticTree:: +Rebalance(HazardNodePtr& u, HazardNodePtr& ux, HazardNodePtr& uxx, + HazardNodePtr& uxxx) { + // Protect node 'u' + PROTECT_NODE_WITH_LOCK(u, u_lock); + // Verify that ux is still a child of u + if (!HasChild(u, ux)) return EMBB_BUSY; + + // Protect node 'ux' + PROTECT_NODE_WITH_LOCK(ux, ux_lock); + // Get children of 'ux' + DEFINE_NODE_WITH_HAZARD(4, uxl, ux, GetLeft); + DEFINE_NODE_WITH_HAZARD(5, uxr, ux, GetRight); + // Verify that 'uxx' is still a child of 'ux' + bool uxx_is_left = (uxx == uxl); (void)uxx_is_left; + if (!HasChild(ux, uxx)) return EMBB_BUSY; + + // Protect node 'uxx' + PROTECT_NODE_WITH_LOCK(uxx, uxx_lock); + // Get children of 'uxx' + DEFINE_NODE_WITH_HAZARD(6, uxxl, uxx, GetLeft); + DEFINE_NODE_WITH_HAZARD(7, uxxr, uxx, GetRight); + // Verify that 'uxxx' is still a child of 'uxx' bool uxxx_is_left = (uxxx == uxxl); - if (!HasChild(uxx, uxxx)) return false; + if (!HasChild(uxx, uxxx)) return EMBB_BUSY; if (uxxx->GetWeight() > 1) { if (uxxx_is_left) { - //TODO: weakLLX(uxxl); - return OverweightLeft(u, ux, uxx, uxl, uxr, uxxl, uxxr, uxx_is_left); + // Protect node 'uxxl' + PROTECT_NODE_WITH_LOCK(uxxl, uxxl_lock); + return OverweightLeft(u, u_lock, ux, ux_lock, uxx, uxx_lock, uxl, uxr, + uxxl, uxxl_lock, uxxr, uxx_is_left); } else { - //TODO: weakLLX(uxxr); - return OverweightRight(u, ux, uxx, uxl, uxr, uxxl, uxxr, !uxx_is_left); + // Protect node 'uxxr' + PROTECT_NODE_WITH_LOCK(uxxr, uxxr_lock); + return OverweightRight(u, u_lock, ux, ux_lock, uxx, uxx_lock, uxl, uxr, + uxxl, uxxr, uxxr_lock, !uxx_is_left); } } else { + assert(uxxx->GetWeight() == 0 && uxx->GetWeight() == 0); //Red-red violation if (uxx_is_left) { if (uxr->GetWeight() == 0) { - //TODO: weakLLX(uxr); - return BLK(u, ux, uxx, uxr); + // Protect node 'uxr' + PROTECT_NODE_WITH_LOCK(uxr, uxr_lock); + return BLK(u, u_lock, ux, ux_lock, uxx, uxx_lock, uxr, uxr_lock); } else if (uxxx_is_left) { - return RB1_L(u, ux, uxx); + return RB1_L(u, u_lock, ux, ux_lock, uxx, uxx_lock); } else { - //TODO: weakLLX(uxxr); - return RB2_L(u, ux, uxx, uxxr); + // Protect node 'uxxr' + PROTECT_NODE_WITH_LOCK(uxxr, uxxr_lock); + return RB2_L(u, u_lock, ux, ux_lock, uxx, uxx_lock, uxxr, uxxr_lock); } } else { if (uxl->GetWeight() == 0) { - //TODO: weakLLX(uxl); - return BLK(u, ux, uxl, uxx); + // Protect node 'uxl' + PROTECT_NODE_WITH_LOCK(uxl, uxl_lock); + return BLK(u, u_lock, ux, ux_lock, uxl, uxl_lock, uxx, uxx_lock); } else if (!uxxx_is_left) { - return RB1_R(u, ux, uxx); + return RB1_R(u, u_lock, ux, ux_lock, uxx, uxx_lock); } else { - //TODO: weakLLX(uxxl); - return RB2_R(u, ux, uxx, uxxl); + // Protect node 'uxxl' + PROTECT_NODE_WITH_LOCK(uxxl, uxxl_lock); + return RB2_R(u, u_lock, ux, ux_lock, uxx, uxx_lock, uxxl, uxxl_lock); } } } } template -bool ChromaticTree:: -OverweightLeft(const NodePtr& u, const NodePtr& ux, const NodePtr& uxx, - const NodePtr& uxl, const NodePtr& uxr, - const NodePtr& uxxl, const NodePtr& uxxr, - const bool& uxx_is_left) { +embb_errors_t ChromaticTree:: +OverweightLeft(HazardNodePtr& u, UniqueLock& u_lock, + HazardNodePtr& ux, UniqueLock& ux_lock, + HazardNodePtr& uxx, UniqueLock& uxx_lock, + HazardNodePtr& uxl, HazardNodePtr& uxr, + HazardNodePtr& uxxl, UniqueLock& uxxl_lock, + HazardNodePtr& uxxr, bool uxx_is_left) { // Let "Root" be the top of the overweight violation decision tree (see p.30) // Root -> Middle if (uxxr->GetWeight() == 0) { @@ -604,14 +617,16 @@ OverweightLeft(const NodePtr& u, const NodePtr& ux, const NodePtr& uxx, if (uxx_is_left) { // Root -> Middle -> Left -> Left -> Left if (uxr->GetWeight() == 0) { - //TODO: weakLLX(uxr); - return BLK(u, ux, uxx, uxr); + // Protect node 'uxr' + PROTECT_NODE_WITH_LOCK(uxr, uxr_lock); + return BLK(u, u_lock, ux, ux_lock, uxx, uxx_lock, uxr, uxr_lock); // Root -> Middle -> Left -> Left -> Right } else { assert(uxr->GetWeight() > 0); - //TODO: weakLLX(uxxr); - return RB2_L(u, ux, uxx, uxxr); + // Protect node 'uxxr' + PROTECT_NODE_WITH_LOCK(uxxr, uxxr_lock); + return RB2_L(u, u_lock, ux, ux_lock, uxx, uxx_lock, uxxr, uxxr_lock); } // Root -> Middle -> Left -> Right @@ -619,102 +634,126 @@ OverweightLeft(const NodePtr& u, const NodePtr& ux, const NodePtr& uxx, assert(!uxx_is_left); // Root -> Middle -> Left -> Right -> Left if (uxl->GetWeight() == 0) { - //TODO: weakLLX(uxl); - return BLK(u, ux, uxl, uxx); + // Protect node 'uxl' + PROTECT_NODE_WITH_LOCK(uxl, uxl_lock); + return BLK(u, u_lock, ux, ux_lock, uxl, uxl_lock, uxx, uxx_lock); // Root -> Middle -> Left -> Right -> Right } else { assert(uxl->GetWeight() > 0); - return RB1_R(u, ux, uxx); + return RB1_R(u, u_lock, ux, ux_lock, uxx, uxx_lock); } } // Root -> Middle -> Right } else { assert(uxx->GetWeight() > 0); - //TODO: weakLLX(uxxr); + // Protect node 'uxxr' + PROTECT_NODE_WITH_LOCK(uxxr, uxxr_lock); + + // Get left child of 'uxxr' // Note: we know that 'uxxr' is not a leaf because it has weight 0. - NodePtr uxxrl = uxxr->GetLeft(); VERIFY_ADDRESS(uxxrl); - //TODO: weakLLX(uxxrl); + DEFINE_NODE_WITH_HAZARD(8, uxxrl, uxxr, GetLeft); + + // Protect node 'uxxrl' + PROTECT_NODE_WITH_LOCK(uxxrl, uxxrl_lock); // Root -> Middle -> Right -> Left if (uxxrl->GetWeight() == 0) { - return RB2_R(ux, uxx, uxxr, uxxrl); + return RB2_R(ux, ux_lock, uxx, uxx_lock, + uxxr, uxxr_lock, uxxrl, uxxrl_lock); // Root -> Middle -> Right -> Middle } else if (uxxrl->GetWeight() == 1) { - NodePtr uxxrll = uxxrl->GetLeft(); VERIFY_ADDRESS(uxxrll); - NodePtr uxxrlr = uxxrl->GetRight(); VERIFY_ADDRESS(uxxrlr); - if (uxxrlr == NULL) return false; + DEFINE_NODE_WITH_HAZARD(9, uxxrlr, uxxrl, GetRight); + if (uxxrlr == NULL) return EMBB_BUSY; // Root -> Middle -> Right -> Middle -> Left if (uxxrlr->GetWeight() == 0) { - //TODO: weakLLX(uxxrlr); - return W4_L(ux, uxx, uxxl, uxxr, uxxrl, uxxrlr); + // Protect node 'uxxrlr' + PROTECT_NODE_WITH_LOCK(uxxrlr, uxxrlr_lock); + return W4_L(ux, ux_lock, uxx, uxx_lock, uxxl, uxxl_lock, + uxxr, uxxr_lock, uxxrl, uxxrl_lock, uxxrlr, uxxrlr_lock); // Root -> Middle -> Right -> Middle -> Right } else { assert(uxxrlr->GetWeight() > 0); // Root -> Middle -> Right -> Middle -> Right -> Left + // Node: reusing hazard of node 'uxxrlr' as it is no longer used + DEFINE_NODE_WITH_HAZARD(9, uxxrll, uxxrl, GetLeft); if (uxxrll->GetWeight() == 0) { - //TODO: weakLLX(uxxrll); - return W3_L(ux, uxx, uxxl, uxxr, uxxrl, uxxrll); + // Protect node 'uxxrll' + PROTECT_NODE_WITH_LOCK(uxxrll, uxxrll_lock); + return W3_L(ux, ux_lock, uxx, uxx_lock, uxxl, uxxl_lock, uxxr, + uxxr_lock, uxxrl, uxxrl_lock, uxxrll, uxxrll_lock); // Root -> Middle -> Right -> Middle -> Right -> Right } else { assert(uxxrll->GetWeight() > 0); - return W2_L(ux, uxx, uxxl, uxxr, uxxrl); + return W2_L(ux, ux_lock, uxx, uxx_lock, uxxl, uxxl_lock, + uxxr, uxxr_lock, uxxrl, uxxrl_lock); } } // Root -> Middle -> Right -> Right } else { assert(uxxrl->GetWeight() > 1); - return W1_L(ux, uxx, uxxl, uxxr, uxxrl); + return W1_L(ux, ux_lock, uxx, uxx_lock, uxxl, uxxl_lock, + uxxr, uxxr_lock, uxxrl, uxxrl_lock); } } // Root -> Right } else if (uxxr->GetWeight() == 1) { - //TODO: weakLLX(uxxr); - NodePtr uxxrl = uxxr->GetLeft(); VERIFY_ADDRESS(uxxrl); - NodePtr uxxrr = uxxr->GetRight(); VERIFY_ADDRESS(uxxrr); - if (uxxrl == NULL) return false; + // Protect node 'uxxr' + PROTECT_NODE_WITH_LOCK(uxxr, uxxr_lock); + // Get children of 'uxxr' + DEFINE_NODE_WITH_HAZARD(8, uxxrl, uxxr, GetLeft); + DEFINE_NODE_WITH_HAZARD(9, uxxrr, uxxr, GetRight); + if (uxxrl == NULL) return EMBB_BUSY; // Root -> Right -> Left if (uxxrr->GetWeight() == 0) { - //TODO: weakLLX(uxxrr); - return W5_L(ux, uxx, uxxl, uxxr, uxxrr); + // Protect node 'uxxrr' + PROTECT_NODE_WITH_LOCK(uxxrr, uxxrr_lock); + return W5_L(ux, ux_lock, uxx, uxx_lock, uxxl, uxxl_lock, + uxxr, uxxr_lock, uxxrr, uxxrr_lock); // Root -> Right -> Right } else { assert(uxxrr->GetWeight() > 0); // Root -> Right -> Right -> Left if (uxxrl->GetWeight() == 0) { - //TODO: weakLLX(uxxrl); - return W6_L(ux, uxx, uxxl, uxxr, uxxrl); + // Protect node 'uxxrl' + PROTECT_NODE_WITH_LOCK(uxxrl, uxxrl_lock); + return W6_L(ux, ux_lock, uxx, uxx_lock, uxxl, uxxl_lock, + uxxr, uxxr_lock, uxxrl, uxxrl_lock); // Root -> Right -> Right -> Right } else { assert(uxxrl->GetWeight() > 0); - return PUSH_L(ux, uxx, uxxl, uxxr); + return PUSH_L(ux, ux_lock, uxx, uxx_lock, + uxxl, uxxl_lock, uxxr, uxxr_lock); } } // Root -> Left } else { assert(uxxr->GetWeight() > 1); - //TODO: weakLLX(uxxr); - return W7(ux, uxx, uxxl, uxxr); + // Protect node 'uxxr' + PROTECT_NODE_WITH_LOCK(uxxr, uxxr_lock); + return W7(ux, ux_lock, uxx, uxx_lock, uxxl, uxxl_lock, uxxr, uxxr_lock); } } template -bool ChromaticTree:: -OverweightRight(const NodePtr& u, const NodePtr& ux, const NodePtr& uxx, - const NodePtr& uxl, const NodePtr& uxr, - const NodePtr& uxxl, const NodePtr& uxxr, - const bool& uxx_is_right) { +embb_errors_t ChromaticTree:: +OverweightRight(HazardNodePtr& u, UniqueLock& u_lock, + HazardNodePtr& ux, UniqueLock& ux_lock, + HazardNodePtr& uxx, UniqueLock& uxx_lock, + HazardNodePtr& uxl, HazardNodePtr& uxr, + HazardNodePtr& uxxl, HazardNodePtr& uxxr, + UniqueLock& uxxr_lock, bool uxx_is_right) { // Let "Root" be the top of the overweight violation decision tree (see p.30) // Root -> Middle if (uxxl->GetWeight() == 0) { @@ -724,14 +763,16 @@ OverweightRight(const NodePtr& u, const NodePtr& ux, const NodePtr& uxx, if (uxx_is_right) { // Root -> Middle -> Left -> Left -> Left if (uxl->GetWeight() == 0) { - //TODO: weakLLX(uxl); - return BLK(u, ux, uxl, uxx); + // Protect node 'uxl' + PROTECT_NODE_WITH_LOCK(uxl, uxl_lock); + return BLK(u, u_lock, ux, ux_lock, uxl, uxl_lock, uxx, uxx_lock); // Root -> Middle -> Left -> Left -> Right } else { assert(uxl->GetWeight() > 0); - //TODO: weakLLX(uxxl); - return RB2_R(u, ux, uxx, uxxl); + // Protect node 'uxxl' + PROTECT_NODE_WITH_LOCK(uxxl, uxxl_lock); + return RB2_R(u, u_lock, ux, ux_lock, uxx, uxx_lock, uxxl, uxxl_lock); } // Root -> Middle -> Left -> Right @@ -739,93 +780,115 @@ OverweightRight(const NodePtr& u, const NodePtr& ux, const NodePtr& uxx, assert(!uxx_is_right); // Root -> Middle -> Left -> Right -> Left if (uxr->GetWeight() == 0) { - //TODO: weakLLX(uxr); - return BLK(u, ux, uxx, uxr); + // Protect node 'uxr' + PROTECT_NODE_WITH_LOCK(uxr, uxr_lock); + return BLK(u, u_lock, ux, ux_lock, uxx, uxx_lock, uxr, uxr_lock); // Root -> Middle -> Left -> Right -> Right } else { assert(uxr->GetWeight() > 0); - return RB1_L(u, ux, uxx); + return RB1_L(u, u_lock, ux, ux_lock, uxx, uxx_lock); } } // Root -> Middle -> Right } else { assert(uxx->GetWeight() > 0); - //TODO: weakLLX(uxxl); + // Protect node 'uxxl' + PROTECT_NODE_WITH_LOCK(uxxl, uxxl_lock); + + // Get left child of 'uxxl' // Note: we know that 'uxxl' is not a leaf because it has weight 0. - NodePtr uxxlr = uxxl->GetRight(); VERIFY_ADDRESS(uxxlr); - //TODO: weakLLX(uxxlr); + DEFINE_NODE_WITH_HAZARD(8, uxxlr, uxxl, GetRight); + + // Protect node 'uxxlr' + PROTECT_NODE_WITH_LOCK(uxxlr, uxxlr_lock); // Root -> Middle -> Right -> Left if (uxxlr->GetWeight() == 0) { - return RB2_L(ux, uxx, uxxl, uxxlr); + return RB2_L(ux, ux_lock, uxx, uxx_lock, + uxxl, uxxl_lock, uxxlr, uxxlr_lock); // Root -> Middle -> Right -> Middle } else if (uxxlr->GetWeight() == 1) { - NodePtr uxxlrl = uxxlr->GetLeft(); VERIFY_ADDRESS(uxxlrl); - NodePtr uxxlrr = uxxlr->GetRight(); VERIFY_ADDRESS(uxxlrr); - if (uxxlrl == NULL) return false; + DEFINE_NODE_WITH_HAZARD(9, uxxlrl, uxxlr, GetLeft); + if (uxxlrl == NULL) return EMBB_BUSY; // Root -> Middle -> Right -> Middle -> Left if (uxxlrl->GetWeight() == 0) { - //TODO: weakLLX(uxxlrl); - return W4_R(ux, uxx, uxxl, uxxr, uxxlr, uxxlrl); + // Protect node 'uxxlrl' + PROTECT_NODE_WITH_LOCK(uxxlrl, uxxlrl_lock); + return W4_R(ux, ux_lock, uxx, uxx_lock, uxxl, uxxl_lock, + uxxr, uxxr_lock, uxxlr, uxxlr_lock, uxxlrl, uxxlrl_lock); // Root -> Middle -> Right -> Middle -> Right } else { assert(uxxlrl->GetWeight() > 0); // Root -> Middle -> Right -> Middle -> Right -> Left + // Node: reusing hazard of node 'uxxlrl' as it is no longer used + DEFINE_NODE_WITH_HAZARD(9, uxxlrr, uxxlr, GetRight); if (uxxlrr->GetWeight() == 0) { - //TODO: weakLLX(uxxlrr); - return W3_R(ux, uxx, uxxl, uxxr, uxxlr, uxxlrr); + // Protect node 'uxxlrr' + PROTECT_NODE_WITH_LOCK(uxxlrr, uxxlrr_lock); + return W3_R(ux, ux_lock, uxx, uxx_lock, uxxl, uxxl_lock, uxxr, + uxxr_lock, uxxlr, uxxlr_lock, uxxlrr, uxxlrr_lock); // Root -> Middle -> Right -> Middle -> Right -> Right } else { assert(uxxlrr->GetWeight() > 0); - return W2_R(ux, uxx, uxxl, uxxr, uxxlr); + return W2_R(ux, ux_lock, uxx, uxx_lock, uxxl, uxxl_lock, + uxxr, uxxr_lock, uxxlr, uxxlr_lock); } } // Root -> Middle -> Right -> Right } else { assert(uxxlr->GetWeight() > 1); - return W1_R(ux, uxx, uxxl, uxxr, uxxlr); + return W1_R(ux, ux_lock, uxx, uxx_lock, uxxl, uxxl_lock, + uxxr, uxxr_lock, uxxlr, uxxlr_lock); } } // Root -> Right } else if (uxxl->GetWeight() == 1) { - //TODO: weakLLX(uxxl); - NodePtr uxxll = uxxl->GetLeft(); VERIFY_ADDRESS(uxxll); - NodePtr uxxlr = uxxl->GetRight(); VERIFY_ADDRESS(uxxlr); - if (uxxll == NULL) return false; + // Protect node 'uxxl' + PROTECT_NODE_WITH_LOCK(uxxl, uxxl_lock); + // Get children of 'uxxl' + DEFINE_NODE_WITH_HAZARD(8, uxxll, uxxl, GetLeft); + DEFINE_NODE_WITH_HAZARD(9, uxxlr, uxxl, GetRight); + if (uxxll == NULL) return EMBB_BUSY; // Root -> Right -> Left if (uxxll->GetWeight() == 0) { - //TODO: weakLLX(uxxll); - return W5_R(ux, uxx, uxxl, uxxr, uxxll); + // Protect node 'uxxll' + PROTECT_NODE_WITH_LOCK(uxxll, uxxll_lock); + return W5_R(ux, ux_lock, uxx, uxx_lock, uxxl, uxxl_lock, + uxxr, uxxr_lock, uxxll, uxxll_lock); // Root -> Right -> Right } else { assert(uxxll->GetWeight() > 0); // Root -> Right -> Right -> Left if (uxxlr->GetWeight() == 0) { - //TODO: weakLLX(uxxlr); - return W6_R(ux, uxx, uxxl, uxxr, uxxlr); + // Protect node 'uxxlr' + PROTECT_NODE_WITH_LOCK(uxxlr, uxxlr_lock); + return W6_R(ux, ux_lock, uxx, uxx_lock, uxxl, uxxl_lock, + uxxr, uxxr_lock, uxxlr, uxxlr_lock); // Root -> Right -> Right -> Right } else { assert(uxxlr->GetWeight() > 0); - return PUSH_R(ux, uxx, uxxl, uxxr); + return PUSH_R(ux, ux_lock, uxx, uxx_lock, + uxxl, uxxl_lock, uxxr, uxxr_lock); } } // Root -> Left } else { assert(uxxl->GetWeight() > 1); - //TODO: weakLLX(uxxl); - return W7(ux, uxx, uxxl, uxxr); + // Protect node 'uxxl' + PROTECT_NODE_WITH_LOCK(uxxl, uxxl_lock); + return W7(ux, ux_lock, uxx, uxx_lock, uxxl, uxxl_lock, uxxr, uxxr_lock); } } diff --git a/containers_cpp/include/embb/containers/internal/lock_free_chromatic_tree-rebalance.h b/containers_cpp/include/embb/containers/internal/lock_free_chromatic_tree-rebalance.h index 5bc0e83..77d34c2 100644 --- a/containers_cpp/include/embb/containers/internal/lock_free_chromatic_tree-rebalance.h +++ b/containers_cpp/include/embb/containers/internal/lock_free_chromatic_tree-rebalance.h @@ -32,10 +32,11 @@ #ifndef EMBB_CONTAINERS_LOCK_FREE_CHROMATIC_TREE_REBALANCE_H_ #define EMBB_CONTAINERS_LOCK_FREE_CHROMATIC_TREE_REBALANCE_H_ -bool BLK(const NodePtr& u, - const NodePtr& ux, - const NodePtr& uxl, - const NodePtr& uxr) { +embb_errors_t BLK( + HazardNodePtr& u, UniqueLock& u_lock, + HazardNodePtr& ux, UniqueLock& ux_lock, + HazardNodePtr& uxl, UniqueLock& uxl_lock, + HazardNodePtr& uxr, UniqueLock& uxr_lock) { NodePtr nxl = node_pool_.Allocate( uxl->GetKey(), uxl->GetValue(), 1, @@ -49,29 +50,34 @@ bool BLK(const NodePtr& u, HasFixedWeight(ux) ? 1 : ux->GetWeight() - 1, nxl, nxr); - if (nxl == NULL || - nxr == NULL || + if (nxl == NULL || + nxr == NULL || nx == NULL) { if (nxl) FreeNode(nxl); if (nxr) FreeNode(nxr); if (nx) FreeNode(nx); - return false; + return EMBB_NOMEM; } NodePtr expected = ux; - GetPointerToChild(u, ux).CompareAndSwap(expected, nx); + bool rotation_succeeded = GetPointerToChild(u, ux) + .CompareAndSwap(expected, nx); + assert(rotation_succeeded); // For now (FGL tree) this CAS may not fail + if (!rotation_succeeded) return EMBB_BUSY; - RetireNode(ux); - RetireNode(uxl); - RetireNode(uxr); + RetireHazardousNode(ux, ux_lock); + RetireHazardousNode(uxl, uxl_lock); + RetireHazardousNode(uxr, uxr_lock); + (void)u_lock; // For now (FGL tree) u_lock is not used here - return true; + return EMBB_SUCCESS; } -bool PUSH_L(const NodePtr& u, - const NodePtr& ux, - const NodePtr& uxl, - const NodePtr& uxr) { +embb_errors_t PUSH_L( + HazardNodePtr& u, UniqueLock& u_lock, + HazardNodePtr& ux, UniqueLock& ux_lock, + HazardNodePtr& uxl, UniqueLock& uxl_lock, + HazardNodePtr& uxr, UniqueLock& uxr_lock) { NodePtr nxl = node_pool_.Allocate( uxl->GetKey(), uxl->GetValue(), uxl->GetWeight() - 1, @@ -85,29 +91,34 @@ bool PUSH_L(const NodePtr& u, HasFixedWeight(ux) ? 1 : ux->GetWeight() + 1, nxl, nxr); - if (nxl == NULL || - nxr == NULL || + if (nxl == NULL || + nxr == NULL || nx == NULL) { if (nxl) FreeNode(nxl); if (nxr) FreeNode(nxr); if (nx) FreeNode(nx); - return false; + return EMBB_NOMEM; } NodePtr expected = ux; - GetPointerToChild(u, ux).CompareAndSwap(expected, nx); + bool rotation_succeeded = GetPointerToChild(u, ux) + .CompareAndSwap(expected, nx); + assert(rotation_succeeded); // For now (FGL tree) this CAS may not fail + if (!rotation_succeeded) return EMBB_BUSY; - RetireNode(ux); - RetireNode(uxl); - RetireNode(uxr); + RetireHazardousNode(ux, ux_lock); + RetireHazardousNode(uxl, uxl_lock); + RetireHazardousNode(uxr, uxr_lock); + (void)u_lock; // For now (FGL tree) u_lock is not used here - return true; + return EMBB_SUCCESS; } -bool PUSH_R(const NodePtr& u, - const NodePtr& ux, - const NodePtr& uxl, - const NodePtr& uxr) { +embb_errors_t PUSH_R( + HazardNodePtr& u, UniqueLock& u_lock, + HazardNodePtr& ux, UniqueLock& ux_lock, + HazardNodePtr& uxl, UniqueLock& uxl_lock, + HazardNodePtr& uxr, UniqueLock& uxr_lock) { NodePtr nxr = node_pool_.Allocate( uxr->GetKey(), uxr->GetValue(), uxr->GetWeight() - 1, @@ -121,28 +132,33 @@ bool PUSH_R(const NodePtr& u, HasFixedWeight(ux) ? 1 : ux->GetWeight() + 1, nxl, nxr); - if (nxr == NULL || - nxl == NULL || + if (nxr == NULL || + nxl == NULL || nx == NULL) { if (nxr) FreeNode(nxr); if (nxl) FreeNode(nxl); if (nx) FreeNode(nx); - return false; + return EMBB_NOMEM; } NodePtr expected = ux; - GetPointerToChild(u, ux).CompareAndSwap(expected, nx); + bool rotation_succeeded = GetPointerToChild(u, ux) + .CompareAndSwap(expected, nx); + assert(rotation_succeeded); // For now (FGL tree) this CAS may not fail + if (!rotation_succeeded) return EMBB_BUSY; - RetireNode(ux); - RetireNode(uxl); - RetireNode(uxr); + RetireHazardousNode(ux, ux_lock); + RetireHazardousNode(uxl, uxl_lock); + RetireHazardousNode(uxr, uxr_lock); + (void)u_lock; // For now (FGL tree) u_lock is not used here - return true; + return EMBB_SUCCESS; } -bool RB1_L(const NodePtr& u, - const NodePtr& ux, - const NodePtr& uxl) { +embb_errors_t RB1_L( + HazardNodePtr& u, UniqueLock& u_lock, + HazardNodePtr& ux, UniqueLock& ux_lock, + HazardNodePtr& uxl, UniqueLock& uxl_lock) { NodePtr nxr = node_pool_.Allocate( ux->GetKey(), ux->GetValue(), 0, @@ -152,25 +168,30 @@ bool RB1_L(const NodePtr& u, ux->GetWeight(), uxl->GetLeft(), nxr); - if (nxr == NULL || + if (nxr == NULL || nx == NULL) { if (nxr) FreeNode(nxr); if (nx) FreeNode(nx); - return false; + return EMBB_NOMEM; } NodePtr expected = ux; - GetPointerToChild(u, ux).CompareAndSwap(expected, nx); + bool rotation_succeeded = GetPointerToChild(u, ux) + .CompareAndSwap(expected, nx); + assert(rotation_succeeded); // For now (FGL tree) this CAS may not fail + if (!rotation_succeeded) return EMBB_BUSY; - RetireNode(ux); - RetireNode(uxl); + RetireHazardousNode(ux, ux_lock); + RetireHazardousNode(uxl, uxl_lock); + (void)u_lock; // For now (FGL tree) u_lock is not used here - return true; + return EMBB_SUCCESS; } -bool RB1_R(const NodePtr& u, - const NodePtr& ux, - const NodePtr& uxr) { +embb_errors_t RB1_R( + HazardNodePtr& u, UniqueLock& u_lock, + HazardNodePtr& ux, UniqueLock& ux_lock, + HazardNodePtr& uxr, UniqueLock& uxr_lock) { NodePtr nxl = node_pool_.Allocate( ux->GetKey(), ux->GetValue(), 0, @@ -180,26 +201,31 @@ bool RB1_R(const NodePtr& u, ux->GetWeight(), nxl, uxr->GetRight()); - if (nxl == NULL || + if (nxl == NULL || nx == NULL) { if (nxl) FreeNode(nxl); if (nx) FreeNode(nx); - return false; + return EMBB_NOMEM; } NodePtr expected = ux; - GetPointerToChild(u, ux).CompareAndSwap(expected, nx); + bool rotation_succeeded = GetPointerToChild(u, ux) + .CompareAndSwap(expected, nx); + assert(rotation_succeeded); // For now (FGL tree) this CAS may not fail + if (!rotation_succeeded) return EMBB_BUSY; - RetireNode(ux); - RetireNode(uxr); + RetireHazardousNode(ux, ux_lock); + RetireHazardousNode(uxr, uxr_lock); + (void)u_lock; // For now (FGL tree) u_lock is not used here - return true; + return EMBB_SUCCESS; } -bool RB2_L(const NodePtr& u, - const NodePtr& ux, - const NodePtr& uxl, - const NodePtr& uxlr) { +embb_errors_t RB2_L( + HazardNodePtr& u, UniqueLock& u_lock, + HazardNodePtr& ux, UniqueLock& ux_lock, + HazardNodePtr& uxl, UniqueLock& uxl_lock, + HazardNodePtr& uxlr, UniqueLock& uxlr_lock) { NodePtr nxl = node_pool_.Allocate( uxl->GetKey(), uxl->GetValue(), 0, @@ -213,29 +239,34 @@ bool RB2_L(const NodePtr& u, ux->GetWeight(), nxl, nxr); - if (nxl == NULL || - nxr == NULL || + if (nxl == NULL || + nxr == NULL || nx == NULL) { if (nxl) FreeNode(nxl); if (nxr) FreeNode(nxr); if (nx) FreeNode(nx); - return false; + return EMBB_NOMEM; } NodePtr expected = ux; - GetPointerToChild(u, ux).CompareAndSwap(expected, nx); + bool rotation_succeeded = GetPointerToChild(u, ux) + .CompareAndSwap(expected, nx); + assert(rotation_succeeded); // For now (FGL tree) this CAS may not fail + if (!rotation_succeeded) return EMBB_BUSY; - RetireNode(ux); - RetireNode(uxl); - RetireNode(uxlr); + RetireHazardousNode(ux, ux_lock); + RetireHazardousNode(uxl, uxl_lock); + RetireHazardousNode(uxlr, uxlr_lock); + (void)u_lock; // For now (FGL tree) u_lock is not used here - return true; + return EMBB_SUCCESS; } -bool RB2_R(const NodePtr& u, - const NodePtr& ux, - const NodePtr& uxr, - const NodePtr& uxrl) { +embb_errors_t RB2_R( + HazardNodePtr& u, UniqueLock& u_lock, + HazardNodePtr& ux, UniqueLock& ux_lock, + HazardNodePtr& uxr, UniqueLock& uxr_lock, + HazardNodePtr& uxrl, UniqueLock& uxrl_lock) { NodePtr nxr = node_pool_.Allocate( uxr->GetKey(), uxr->GetValue(), 0, @@ -249,30 +280,35 @@ bool RB2_R(const NodePtr& u, ux->GetWeight(), nxl, nxr); - if (nxr == NULL || - nxl == NULL || + if (nxr == NULL || + nxl == NULL || nx == NULL) { if (nxr) FreeNode(nxr); if (nxl) FreeNode(nxl); if (nx) FreeNode(nx); - return false; + return EMBB_NOMEM; } NodePtr expected = ux; - GetPointerToChild(u, ux).CompareAndSwap(expected, nx); + bool rotation_succeeded = GetPointerToChild(u, ux) + .CompareAndSwap(expected, nx); + assert(rotation_succeeded); // For now (FGL tree) this CAS may not fail + if (!rotation_succeeded) return EMBB_BUSY; - RetireNode(ux); - RetireNode(uxr); - RetireNode(uxrl); + RetireHazardousNode(ux, ux_lock); + RetireHazardousNode(uxr, uxr_lock); + RetireHazardousNode(uxrl, uxrl_lock); + (void)u_lock; // For now (FGL tree) u_lock is not used here - return true; + return EMBB_SUCCESS; } -bool W1_L(const NodePtr& u, - const NodePtr& ux, - const NodePtr& uxl, - const NodePtr& uxr, - const NodePtr& uxrl) { +embb_errors_t W1_L( + HazardNodePtr& u, UniqueLock& u_lock, + HazardNodePtr& ux, UniqueLock& ux_lock, + HazardNodePtr& uxl, UniqueLock& uxl_lock, + HazardNodePtr& uxr, UniqueLock& uxr_lock, + HazardNodePtr& uxrl, UniqueLock& uxrl_lock) { NodePtr nxll = node_pool_.Allocate( uxl->GetKey(), uxl->GetValue(), uxl->GetWeight() - 1, @@ -290,33 +326,38 @@ bool W1_L(const NodePtr& u, ux->GetWeight(), nxl, uxr->GetRight()); - if (nxll == NULL || - nxlr == NULL || - nxl == NULL || + if (nxll == NULL || + nxlr == NULL || + nxl == NULL || nx == NULL) { if (nxll) FreeNode(nxll); if (nxlr) FreeNode(nxlr); if (nxl) FreeNode(nxl); if (nx) FreeNode(nx); - return false; + return EMBB_NOMEM; } NodePtr expected = ux; - GetPointerToChild(u, ux).CompareAndSwap(expected, nx); - - RetireNode(ux); - RetireNode(uxl); - RetireNode(uxr); - RetireNode(uxrl); - - return true; + bool rotation_succeeded = GetPointerToChild(u, ux) + .CompareAndSwap(expected, nx); + assert(rotation_succeeded); // For now (FGL tree) this CAS may not fail + if (!rotation_succeeded) return EMBB_BUSY; + + RetireHazardousNode(ux, ux_lock); + RetireHazardousNode(uxl, uxl_lock); + RetireHazardousNode(uxr, uxr_lock); + RetireHazardousNode(uxrl, uxrl_lock); + (void)u_lock; // For now (FGL tree) u_lock is not used here + + return EMBB_SUCCESS; } -bool W1_R(const NodePtr& u, - const NodePtr& ux, - const NodePtr& uxl, - const NodePtr& uxr, - const NodePtr& uxlr) { +embb_errors_t W1_R( + HazardNodePtr& u, UniqueLock& u_lock, + HazardNodePtr& ux, UniqueLock& ux_lock, + HazardNodePtr& uxl, UniqueLock& uxl_lock, + HazardNodePtr& uxr, UniqueLock& uxr_lock, + HazardNodePtr& uxlr, UniqueLock& uxlr_lock) { NodePtr nxrr = node_pool_.Allocate( uxr->GetKey(), uxr->GetValue(), uxr->GetWeight() - 1, @@ -334,33 +375,38 @@ bool W1_R(const NodePtr& u, ux->GetWeight(), uxl->GetLeft(), nxr); - if (nxrr == NULL || - nxrl == NULL || - nxr == NULL || + if (nxrr == NULL || + nxrl == NULL || + nxr == NULL || nx == NULL) { if (nxrr) FreeNode(nxrr); if (nxrl) FreeNode(nxrl); if (nxr) FreeNode(nxr); if (nx) FreeNode(nx); - return false; + return EMBB_NOMEM; } NodePtr expected = ux; - GetPointerToChild(u, ux).CompareAndSwap(expected, nx); - - RetireNode(ux); - RetireNode(uxl); - RetireNode(uxr); - RetireNode(uxlr); - - return true; + bool rotation_succeeded = GetPointerToChild(u, ux) + .CompareAndSwap(expected, nx); + assert(rotation_succeeded); // For now (FGL tree) this CAS may not fail + if (!rotation_succeeded) return EMBB_BUSY; + + RetireHazardousNode(ux, ux_lock); + RetireHazardousNode(uxl, uxl_lock); + RetireHazardousNode(uxr, uxr_lock); + RetireHazardousNode(uxlr, uxlr_lock); + (void)u_lock; // For now (FGL tree) u_lock is not used here + + return EMBB_SUCCESS; } -bool W2_L(const NodePtr& u, - const NodePtr& ux, - const NodePtr& uxl, - const NodePtr& uxr, - const NodePtr& uxrl) { +embb_errors_t W2_L( + HazardNodePtr& u, UniqueLock& u_lock, + HazardNodePtr& ux, UniqueLock& ux_lock, + HazardNodePtr& uxl, UniqueLock& uxl_lock, + HazardNodePtr& uxr, UniqueLock& uxr_lock, + HazardNodePtr& uxrl, UniqueLock& uxrl_lock) { NodePtr nxll = node_pool_.Allocate( uxl->GetKey(), uxl->GetValue(), uxl->GetWeight() - 1, @@ -378,33 +424,38 @@ bool W2_L(const NodePtr& u, ux->GetWeight(), nxl, uxr->GetRight()); - if (nxll == NULL || - nxlr == NULL || - nxl == NULL || + if (nxll == NULL || + nxlr == NULL || + nxl == NULL || nx == NULL) { if (nxll) FreeNode(nxll); if (nxlr) FreeNode(nxlr); if (nxl) FreeNode(nxl); if (nx) FreeNode(nx); - return false; + return EMBB_NOMEM; } NodePtr expected = ux; - GetPointerToChild(u, ux).CompareAndSwap(expected, nx); - - RetireNode(ux); - RetireNode(uxl); - RetireNode(uxr); - RetireNode(uxrl); - - return true; + bool rotation_succeeded = GetPointerToChild(u, ux) + .CompareAndSwap(expected, nx); + assert(rotation_succeeded); // For now (FGL tree) this CAS may not fail + if (!rotation_succeeded) return EMBB_BUSY; + + RetireHazardousNode(ux, ux_lock); + RetireHazardousNode(uxl, uxl_lock); + RetireHazardousNode(uxr, uxr_lock); + RetireHazardousNode(uxrl, uxrl_lock); + (void)u_lock; // For now (FGL tree) u_lock is not used here + + return EMBB_SUCCESS; } -bool W2_R(const NodePtr& u, - const NodePtr& ux, - const NodePtr& uxl, - const NodePtr& uxr, - const NodePtr& uxlr) { +embb_errors_t W2_R( + HazardNodePtr& u, UniqueLock& u_lock, + HazardNodePtr& ux, UniqueLock& ux_lock, + HazardNodePtr& uxl, UniqueLock& uxl_lock, + HazardNodePtr& uxr, UniqueLock& uxr_lock, + HazardNodePtr& uxlr, UniqueLock& uxlr_lock) { NodePtr nxrr = node_pool_.Allocate( uxr->GetKey(), uxr->GetValue(), uxr->GetWeight() - 1, @@ -422,34 +473,39 @@ bool W2_R(const NodePtr& u, ux->GetWeight(), uxl->GetLeft(), nxr); - if (nxrr == NULL || - nxrl == NULL || - nxr == NULL || + if (nxrr == NULL || + nxrl == NULL || + nxr == NULL || nx == NULL) { if (nxrr) FreeNode(nxrr); if (nxrl) FreeNode(nxrl); if (nxr) FreeNode(nxr); if (nx) FreeNode(nx); - return false; + return EMBB_NOMEM; } NodePtr expected = ux; - GetPointerToChild(u, ux).CompareAndSwap(expected, nx); - - RetireNode(ux); - RetireNode(uxl); - RetireNode(uxr); - RetireNode(uxlr); - - return true; + bool rotation_succeeded = GetPointerToChild(u, ux) + .CompareAndSwap(expected, nx); + assert(rotation_succeeded); // For now (FGL tree) this CAS may not fail + if (!rotation_succeeded) return EMBB_BUSY; + + RetireHazardousNode(ux, ux_lock); + RetireHazardousNode(uxl, uxl_lock); + RetireHazardousNode(uxr, uxr_lock); + RetireHazardousNode(uxlr, uxlr_lock); + (void)u_lock; // For now (FGL tree) u_lock is not used here + + return EMBB_SUCCESS; } -bool W3_L(const NodePtr& u, - const NodePtr& ux, - const NodePtr& uxl, - const NodePtr& uxr, - const NodePtr& uxrl, - const NodePtr& uxrll) { +embb_errors_t W3_L( + HazardNodePtr& u, UniqueLock& u_lock, + HazardNodePtr& ux, UniqueLock& ux_lock, + HazardNodePtr& uxl, UniqueLock& uxl_lock, + HazardNodePtr& uxr, UniqueLock& uxr_lock, + HazardNodePtr& uxrl, UniqueLock& uxrl_lock, + HazardNodePtr& uxrll, UniqueLock& uxrll_lock) { NodePtr nxlll = node_pool_.Allocate( uxl->GetKey(), uxl->GetValue(), uxl->GetWeight() - 1, @@ -471,37 +527,42 @@ bool W3_L(const NodePtr& u, ux->GetWeight(), nxl, uxr->GetRight()); - if (nxlll == NULL || - nxll == NULL || - nxlr == NULL || - nxl == NULL || + if (nxlll == NULL || + nxll == NULL || + nxlr == NULL || + nxl == NULL || nx == NULL) { if (nxlll) FreeNode(nxlll); if (nxll) FreeNode(nxll); if (nxlr) FreeNode(nxlr); if (nxl) FreeNode(nxl); if (nx) FreeNode(nx); - return false; + return EMBB_NOMEM; } NodePtr expected = ux; - GetPointerToChild(u, ux).CompareAndSwap(expected, nx); - - RetireNode(ux); - RetireNode(uxl); - RetireNode(uxr); - RetireNode(uxrl); - RetireNode(uxrll); - - return true; + bool rotation_succeeded = GetPointerToChild(u, ux) + .CompareAndSwap(expected, nx); + assert(rotation_succeeded); // For now (FGL tree) this CAS may not fail + if (!rotation_succeeded) return EMBB_BUSY; + + RetireHazardousNode(ux, ux_lock); + RetireHazardousNode(uxl, uxl_lock); + RetireHazardousNode(uxr, uxr_lock); + RetireHazardousNode(uxrl, uxrl_lock); + RetireHazardousNode(uxrll, uxrll_lock); + (void)u_lock; // For now (FGL tree) u_lock is not used here + + return EMBB_SUCCESS; } -bool W3_R(const NodePtr& u, - const NodePtr& ux, - const NodePtr& uxl, - const NodePtr& uxr, - const NodePtr& uxlr, - const NodePtr& uxlrr) { +embb_errors_t W3_R( + HazardNodePtr& u, UniqueLock& u_lock, + HazardNodePtr& ux, UniqueLock& ux_lock, + HazardNodePtr& uxl, UniqueLock& uxl_lock, + HazardNodePtr& uxr, UniqueLock& uxr_lock, + HazardNodePtr& uxlr, UniqueLock& uxlr_lock, + HazardNodePtr& uxlrr, UniqueLock& uxlrr_lock) { NodePtr nxrrr = node_pool_.Allocate( uxr->GetKey(), uxr->GetValue(), uxr->GetWeight() - 1, @@ -523,37 +584,42 @@ bool W3_R(const NodePtr& u, ux->GetWeight(), uxl->GetLeft(), nxr); - if (nxrrr == NULL || - nxrr == NULL || - nxrl == NULL || - nxr == NULL || + if (nxrrr == NULL || + nxrr == NULL || + nxrl == NULL || + nxr == NULL || nx == NULL) { if (nxrrr) FreeNode(nxrrr); if (nxrr) FreeNode(nxrr); if (nxrl) FreeNode(nxrl); if (nxr) FreeNode(nxr); if (nx) FreeNode(nx); - return false; + return EMBB_NOMEM; } NodePtr expected = ux; - GetPointerToChild(u, ux).CompareAndSwap(expected, nx); - - RetireNode(ux); - RetireNode(uxl); - RetireNode(uxr); - RetireNode(uxlr); - RetireNode(uxlrr); - - return true; + bool rotation_succeeded = GetPointerToChild(u, ux) + .CompareAndSwap(expected, nx); + assert(rotation_succeeded); // For now (FGL tree) this CAS may not fail + if (!rotation_succeeded) return EMBB_BUSY; + + RetireHazardousNode(ux, ux_lock); + RetireHazardousNode(uxl, uxl_lock); + RetireHazardousNode(uxr, uxr_lock); + RetireHazardousNode(uxlr, uxlr_lock); + RetireHazardousNode(uxlrr, uxlrr_lock); + (void)u_lock; // For now (FGL tree) u_lock is not used here + + return EMBB_SUCCESS; } -bool W4_L(const NodePtr& u, - const NodePtr& ux, - const NodePtr& uxl, - const NodePtr& uxr, - const NodePtr& uxrl, - const NodePtr& uxrlr) { +embb_errors_t W4_L( + HazardNodePtr& u, UniqueLock& u_lock, + HazardNodePtr& ux, UniqueLock& ux_lock, + HazardNodePtr& uxl, UniqueLock& uxl_lock, + HazardNodePtr& uxr, UniqueLock& uxr_lock, + HazardNodePtr& uxrl, UniqueLock& uxrl_lock, + HazardNodePtr& uxrlr, UniqueLock& uxrlr_lock) { NodePtr nxll = node_pool_.Allocate( uxl->GetKey(), uxl->GetValue(), uxl->GetWeight() - 1, @@ -575,37 +641,42 @@ bool W4_L(const NodePtr& u, ux->GetWeight(), nxl, nxr); - if (nxll == NULL || - nxrl == NULL || - nxl == NULL || - nxr == NULL || + if (nxll == NULL || + nxrl == NULL || + nxl == NULL || + nxr == NULL || nx == NULL) { if (nxll) FreeNode(nxll); if (nxrl) FreeNode(nxrl); if (nxl) FreeNode(nxl); if (nxr) FreeNode(nxr); if (nx) FreeNode(nx); - return false; + return EMBB_NOMEM; } NodePtr expected = ux; - GetPointerToChild(u, ux).CompareAndSwap(expected, nx); - - RetireNode(ux); - RetireNode(uxl); - RetireNode(uxr); - RetireNode(uxrl); - RetireNode(uxrlr); - - return true; + bool rotation_succeeded = GetPointerToChild(u, ux) + .CompareAndSwap(expected, nx); + assert(rotation_succeeded); // For now (FGL tree) this CAS may not fail + if (!rotation_succeeded) return EMBB_BUSY; + + RetireHazardousNode(ux, ux_lock); + RetireHazardousNode(uxl, uxl_lock); + RetireHazardousNode(uxr, uxr_lock); + RetireHazardousNode(uxrl, uxrl_lock); + RetireHazardousNode(uxrlr, uxrlr_lock); + (void)u_lock; // For now (FGL tree) u_lock is not used here + + return EMBB_SUCCESS; } -bool W4_R(const NodePtr& u, - const NodePtr& ux, - const NodePtr& uxl, - const NodePtr& uxr, - const NodePtr& uxlr, - const NodePtr& uxlrl) { +embb_errors_t W4_R( + HazardNodePtr& u, UniqueLock& u_lock, + HazardNodePtr& ux, UniqueLock& ux_lock, + HazardNodePtr& uxl, UniqueLock& uxl_lock, + HazardNodePtr& uxr, UniqueLock& uxr_lock, + HazardNodePtr& uxlr, UniqueLock& uxlr_lock, + HazardNodePtr& uxlrl, UniqueLock& uxlrl_lock) { NodePtr nxrr = node_pool_.Allocate( uxr->GetKey(), uxr->GetValue(), uxr->GetWeight() - 1, @@ -627,36 +698,41 @@ bool W4_R(const NodePtr& u, ux->GetWeight(), nxl, nxr); - if (nxrr == NULL || - nxlr == NULL || - nxr == NULL || - nxl == NULL || + if (nxrr == NULL || + nxlr == NULL || + nxr == NULL || + nxl == NULL || nx == NULL) { if (nxrr) FreeNode(nxrr); if (nxlr) FreeNode(nxlr); if (nxr) FreeNode(nxr); if (nxl) FreeNode(nxl); if (nx) FreeNode(nx); - return false; + return EMBB_NOMEM; } NodePtr expected = ux; - GetPointerToChild(u, ux).CompareAndSwap(expected, nx); - - RetireNode(ux); - RetireNode(uxl); - RetireNode(uxr); - RetireNode(uxlr); - RetireNode(uxlrl); - - return true; + bool rotation_succeeded = GetPointerToChild(u, ux) + .CompareAndSwap(expected, nx); + assert(rotation_succeeded); // For now (FGL tree) this CAS may not fail + if (!rotation_succeeded) return EMBB_BUSY; + + RetireHazardousNode(ux, ux_lock); + RetireHazardousNode(uxl, uxl_lock); + RetireHazardousNode(uxr, uxr_lock); + RetireHazardousNode(uxlr, uxlr_lock); + RetireHazardousNode(uxlrl, uxlrl_lock); + (void)u_lock; // For now (FGL tree) u_lock is not used here + + return EMBB_SUCCESS; } -bool W5_L(const NodePtr& u, - const NodePtr& ux, - const NodePtr& uxl, - const NodePtr& uxr, - const NodePtr& uxrr) { +embb_errors_t W5_L( + HazardNodePtr& u, UniqueLock& u_lock, + HazardNodePtr& ux, UniqueLock& ux_lock, + HazardNodePtr& uxl, UniqueLock& uxl_lock, + HazardNodePtr& uxr, UniqueLock& uxr_lock, + HazardNodePtr& uxrr, UniqueLock& uxrr_lock) { NodePtr nxll = node_pool_.Allocate( uxl->GetKey(), uxl->GetValue(), uxl->GetWeight() - 1, @@ -674,33 +750,38 @@ bool W5_L(const NodePtr& u, ux->GetWeight(), nxl, nxr); - if (nxll == NULL || - nxl == NULL || - nxr == NULL || + if (nxll == NULL || + nxl == NULL || + nxr == NULL || nx == NULL) { if (nxll) FreeNode(nxll); if (nxl) FreeNode(nxl); if (nxr) FreeNode(nxr); if (nx) FreeNode(nx); - return false; + return EMBB_NOMEM; } NodePtr expected = ux; - GetPointerToChild(u, ux).CompareAndSwap(expected, nx); - - RetireNode(ux); - RetireNode(uxl); - RetireNode(uxr); - RetireNode(uxrr); - - return true; + bool rotation_succeeded = GetPointerToChild(u, ux) + .CompareAndSwap(expected, nx); + assert(rotation_succeeded); // For now (FGL tree) this CAS may not fail + if (!rotation_succeeded) return EMBB_BUSY; + + RetireHazardousNode(ux, ux_lock); + RetireHazardousNode(uxl, uxl_lock); + RetireHazardousNode(uxr, uxr_lock); + RetireHazardousNode(uxrr, uxrr_lock); + (void)u_lock; // For now (FGL tree) u_lock is not used here + + return EMBB_SUCCESS; } -bool W5_R(const NodePtr& u, - const NodePtr& ux, - const NodePtr& uxl, - const NodePtr& uxr, - const NodePtr& uxll) { +embb_errors_t W5_R( + HazardNodePtr& u, UniqueLock& u_lock, + HazardNodePtr& ux, UniqueLock& ux_lock, + HazardNodePtr& uxl, UniqueLock& uxl_lock, + HazardNodePtr& uxr, UniqueLock& uxr_lock, + HazardNodePtr& uxll, UniqueLock& uxll_lock) { NodePtr nxrr = node_pool_.Allocate( uxr->GetKey(), uxr->GetValue(), uxr->GetWeight() - 1, @@ -718,33 +799,38 @@ bool W5_R(const NodePtr& u, ux->GetWeight(), nxl, nxr); - if (nxrr == NULL || - nxr == NULL || - nxl == NULL || + if (nxrr == NULL || + nxr == NULL || + nxl == NULL || nx == NULL) { if (nxrr) FreeNode(nxrr); if (nxr) FreeNode(nxr); if (nxl) FreeNode(nxl); if (nx) FreeNode(nx); - return false; + return EMBB_NOMEM; } NodePtr expected = ux; - GetPointerToChild(u, ux).CompareAndSwap(expected, nx); - - RetireNode(ux); - RetireNode(uxl); - RetireNode(uxr); - RetireNode(uxll); - - return true; + bool rotation_succeeded = GetPointerToChild(u, ux) + .CompareAndSwap(expected, nx); + assert(rotation_succeeded); // For now (FGL tree) this CAS may not fail + if (!rotation_succeeded) return EMBB_BUSY; + + RetireHazardousNode(ux, ux_lock); + RetireHazardousNode(uxl, uxl_lock); + RetireHazardousNode(uxr, uxr_lock); + RetireHazardousNode(uxll, uxll_lock); + (void)u_lock; // For now (FGL tree) u_lock is not used here + + return EMBB_SUCCESS; } -bool W6_L(const NodePtr& u, - const NodePtr& ux, - const NodePtr& uxl, - const NodePtr& uxr, - const NodePtr& uxrl) { +embb_errors_t W6_L( + HazardNodePtr& u, UniqueLock& u_lock, + HazardNodePtr& ux, UniqueLock& ux_lock, + HazardNodePtr& uxl, UniqueLock& uxl_lock, + HazardNodePtr& uxr, UniqueLock& uxr_lock, + HazardNodePtr& uxrl, UniqueLock& uxrl_lock) { NodePtr nxll = node_pool_.Allocate( uxl->GetKey(), uxl->GetValue(), uxl->GetWeight() - 1, @@ -762,33 +848,38 @@ bool W6_L(const NodePtr& u, ux->GetWeight(), nxl, nxr); - if (nxll == NULL || - nxl == NULL || - nxr == NULL || + if (nxll == NULL || + nxl == NULL || + nxr == NULL || nx == NULL) { if (nxll) FreeNode(nxll); if (nxl) FreeNode(nxl); if (nxr) FreeNode(nxr); if (nx) FreeNode(nx); - return false; + return EMBB_NOMEM; } NodePtr expected = ux; - GetPointerToChild(u, ux).CompareAndSwap(expected, nx); - - RetireNode(ux); - RetireNode(uxl); - RetireNode(uxr); - RetireNode(uxrl); - - return true; + bool rotation_succeeded = GetPointerToChild(u, ux) + .CompareAndSwap(expected, nx); + assert(rotation_succeeded); // For now (FGL tree) this CAS may not fail + if (!rotation_succeeded) return EMBB_BUSY; + + RetireHazardousNode(ux, ux_lock); + RetireHazardousNode(uxl, uxl_lock); + RetireHazardousNode(uxr, uxr_lock); + RetireHazardousNode(uxrl, uxrl_lock); + (void)u_lock; // For now (FGL tree) u_lock is not used here + + return EMBB_SUCCESS; } -bool W6_R(const NodePtr& u, - const NodePtr& ux, - const NodePtr& uxl, - const NodePtr& uxr, - const NodePtr& uxlr) { +embb_errors_t W6_R( + HazardNodePtr& u, UniqueLock& u_lock, + HazardNodePtr& ux, UniqueLock& ux_lock, + HazardNodePtr& uxl, UniqueLock& uxl_lock, + HazardNodePtr& uxr, UniqueLock& uxr_lock, + HazardNodePtr& uxlr, UniqueLock& uxlr_lock) { NodePtr nxrr = node_pool_.Allocate( uxr->GetKey(), uxr->GetValue(), uxr->GetWeight() - 1, @@ -806,32 +897,37 @@ bool W6_R(const NodePtr& u, ux->GetWeight(), nxl, nxr); - if (nxrr == NULL || - nxr == NULL || - nxl == NULL || + if (nxrr == NULL || + nxr == NULL || + nxl == NULL || nx == NULL) { if (nxrr) FreeNode(nxrr); if (nxr) FreeNode(nxr); if (nxl) FreeNode(nxl); if (nx) FreeNode(nx); - return false; + return EMBB_NOMEM; } NodePtr expected = ux; - GetPointerToChild(u, ux).CompareAndSwap(expected, nx); - - RetireNode(ux); - RetireNode(uxl); - RetireNode(uxr); - RetireNode(uxlr); - - return true; + bool rotation_succeeded = GetPointerToChild(u, ux) + .CompareAndSwap(expected, nx); + assert(rotation_succeeded); // For now (FGL tree) this CAS may not fail + if (!rotation_succeeded) return EMBB_BUSY; + + RetireHazardousNode(ux, ux_lock); + RetireHazardousNode(uxl, uxl_lock); + RetireHazardousNode(uxr, uxr_lock); + RetireHazardousNode(uxlr, uxlr_lock); + (void)u_lock; // For now (FGL tree) u_lock is not used here + + return EMBB_SUCCESS; } -bool W7(const NodePtr& u, - const NodePtr& ux, - const NodePtr& uxl, - const NodePtr& uxr) { +embb_errors_t W7( + HazardNodePtr& u, UniqueLock& u_lock, + HazardNodePtr& ux, UniqueLock& ux_lock, + HazardNodePtr& uxl, UniqueLock& uxl_lock, + HazardNodePtr& uxr, UniqueLock& uxr_lock) { NodePtr nxl = node_pool_.Allocate( uxl->GetKey(), uxl->GetValue(), uxl->GetWeight() - 1, @@ -845,23 +941,27 @@ bool W7(const NodePtr& u, HasFixedWeight(ux) ? 1 : ux->GetWeight() + 1, nxl, nxr); - if (nxl == NULL || - nxr == NULL || + if (nxl == NULL || + nxr == NULL || nx == NULL) { if (nxl) FreeNode(nxl); if (nxr) FreeNode(nxr); if (nx) FreeNode(nx); - return false; + return EMBB_NOMEM; } NodePtr expected = ux; - GetPointerToChild(u, ux).CompareAndSwap(expected, nx); + bool rotation_succeeded = GetPointerToChild(u, ux) + .CompareAndSwap(expected, nx); + assert(rotation_succeeded); // For now (FGL tree) this CAS may not fail + if (!rotation_succeeded) return EMBB_BUSY; - RetireNode(ux); - RetireNode(uxl); - RetireNode(uxr); + RetireHazardousNode(ux, ux_lock); + RetireHazardousNode(uxl, uxl_lock); + RetireHazardousNode(uxr, uxr_lock); + (void)u_lock; // For now (FGL tree) u_lock is not used here - return true; + return EMBB_SUCCESS; } #endif // EMBB_CONTAINERS_LOCK_FREE_CHROMATIC_TREE_REBALANCE_H_ diff --git a/containers_cpp/include/embb/containers/lock_free_chromatic_tree.h b/containers_cpp/include/embb/containers/lock_free_chromatic_tree.h index c431c8d..4ef178e 100644 --- a/containers_cpp/include/embb/containers/lock_free_chromatic_tree.h +++ b/containers_cpp/include/embb/containers/lock_free_chromatic_tree.h @@ -30,6 +30,7 @@ #include #include +#include #include #include @@ -70,6 +71,7 @@ class ChromaticTreeNode { * * \param[IN] key Key of the new node * \param[IN] value Value of the new node + * \param[IN] weight Weight of the new node */ ChromaticTreeNode(const Key& key, const Value& value, const int weight = 1); @@ -126,6 +128,75 @@ class ChromaticTreeNode { embb::base::Mutex mutex_; }; +template +class UniqueHazardGuard { + public: + typedef embb::base::Atomic AtomicGuard; + + UniqueHazardGuard(AtomicGuard& guard, const GuardedType& undefined_guard) + : guard_(guard), + undefined_guard_(undefined_guard), + active_(guard_.Load() == undefined_guard_) {} + + ~UniqueHazardGuard() { + if (active_) SetUndefinedGuard(); + } + + bool ProtectHazard(const AtomicGuard& hazard) { + // Read the hazard and store it into the guard + guard_ = hazard.Load(); + + // Check whether the guard is valid + active_ = (guard_.Load() == hazard.Load()); + + // Clear the guard if it is invalid + if (!active_) SetUndefinedGuard(); + + return active_; + } + + bool IsActive() const { + return active_; + } + + operator GuardedType () const { + assert(active_ == true); + return guard_.Load(); + } + + GuardedType operator->() const { + assert(active_ == true); + return guard_.Load(); + } + + void AdoptGuard(const UniqueHazardGuard& other) { + guard_ = other.guard_.Load(); + active_ = other.active_; + } + + GuardedType ReleaseHazard() { + assert(active_ == true); + GuardedType released_hazard = guard_.Load(); + SetUndefinedGuard(); + active_ = false; + return released_hazard; + } + + private: + void SetUndefinedGuard() { + guard_ = undefined_guard_; + } + + // Non-copyable + UniqueHazardGuard(const UniqueHazardGuard&); + UniqueHazardGuard& + operator=(const UniqueHazardGuard&); + + AtomicGuard& guard_; + GuardedType undefined_guard_; + bool active_; +}; + } // namespace internal namespace test { @@ -301,29 +372,20 @@ class ChromaticTree { */ typedef embb::base::Atomic AtomicNodePtr; + typedef internal::UniqueHazardGuard HazardNodePtr; typedef embb::base::UniqueLock UniqueLock; /** * Follows a path from the root of the tree to some leaf searching for the * given key (the leaf found by this method may or may not contain the given - * key). Returns the reached leaf. - * - * \param[IN] key Key to be searched for - * \param[IN,OUT] leaf Reference to the reached leaf - */ - void Search(const Key& key, NodePtr& leaf); - - /** - * Follows a path from the root of the tree to some leaf searching for the - * given key (the leaf found by this method may or may not contain the given * key). Returns the reached leaf together with its ancestors. * * \param[IN] key Key to be searched for * \param[IN,OUT] leaf Reference to the reached leaf * \param[IN,OUT] parent Reference to the parent of the reached leaf */ - void Search(const Key& key, NodePtr& leaf, NodePtr& parent); + void Search(const Key& key, HazardNodePtr& leaf, HazardNodePtr& parent); /** * Follows a path from the root of the tree to some leaf searching for the @@ -335,8 +397,8 @@ class ChromaticTree { * \param[IN,OUT] parent Reference to the parent of the reached leaf * \param[IN,OUT] grandparent Reference to the grandparent of the reached leaf */ - void Search(const Key& key, NodePtr& leaf, NodePtr& parent, - NodePtr& grandparent); + void Search(const Key& key, HazardNodePtr& leaf, HazardNodePtr& parent, + HazardNodePtr& grandparent); /** * Checks whether the given node is a leaf. @@ -415,8 +477,11 @@ class ChromaticTree { bool IsBalanced() const; bool IsBalanced(const NodePtr& node) const; - void RemoveNode(const NodePtr& node) { - node_hazard_manager_.EnqueuePointerForDeletion(node); + void RetireHazardousNode(HazardNodePtr& node, UniqueLock& node_lock) { + node->Retire(); + node_lock.Unlock(); + NodePtr node_to_delete = node.ReleaseHazard(); + node_hazard_manager_.EnqueuePointerForDeletion(node_to_delete); } /** @@ -440,16 +505,22 @@ class ChromaticTree { /** * Next block of methods is used internally to keep the balance of the tree. */ - bool Rebalance(const NodePtr& u, const NodePtr& ux, const NodePtr& uxx, - const NodePtr& uxxx); - bool OverweightLeft(const NodePtr& u, const NodePtr& ux, const NodePtr& uxx, - const NodePtr& uxl, const NodePtr& uxr, - const NodePtr& uxxl, const NodePtr& uxxr, - const bool& uxx_is_left); - bool OverweightRight(const NodePtr& u, const NodePtr& ux, const NodePtr& uxx, - const NodePtr& uxl, const NodePtr& uxr, - const NodePtr& uxxl, const NodePtr& uxxr, - const bool& uxx_is_right); + embb_errors_t Rebalance(HazardNodePtr& u, HazardNodePtr& ux, + HazardNodePtr& uxx, HazardNodePtr& uxxx); + + embb_errors_t OverweightLeft(HazardNodePtr& u, UniqueLock& u_lock, + HazardNodePtr& ux, UniqueLock& ux_lock, + HazardNodePtr& uxx, UniqueLock& uxx_lock, + HazardNodePtr& uxl, HazardNodePtr& uxr, + HazardNodePtr& uxxl, UniqueLock& uxxl_lock, + HazardNodePtr& uxxr, bool uxx_is_left); + + embb_errors_t OverweightRight(HazardNodePtr& u, UniqueLock& u_lock, + HazardNodePtr& ux, UniqueLock& ux_lock, + HazardNodePtr& uxx, UniqueLock& uxx_lock, + HazardNodePtr& uxl, HazardNodePtr& uxr, + HazardNodePtr& uxxl, HazardNodePtr& uxxr, + UniqueLock& uxxr_lock, bool uxx_is_right); // The following included header contains the class methods implementing // tree rotations. It is generated automatically and must be included @@ -466,7 +537,7 @@ class ChromaticTree { const Compare compare_; /**< Comparator object for the keys */ size_t capacity_; /**< User-requested capacity of the tree */ NodePool node_pool_; /**< Comparator object for the keys */ - NodePtr entry_; /**< Pointer to the sentinel node used as + AtomicNodePtr entry_; /**< Pointer to the sentinel node used as * the entry point into the tree */ /** diff --git a/containers_cpp/test/tree_test-inl.h b/containers_cpp/test/tree_test-inl.h index 8e923a4..0873444 100644 --- a/containers_cpp/test/tree_test-inl.h +++ b/containers_cpp/test/tree_test-inl.h @@ -66,11 +66,11 @@ TreeTest::TreeTest() Add(&TreeTest::TreeTestConcurrentGet_ReaderMethod, this, NUM_TEST_THREADS / 2, NUM_ITERATIONS). Post(&TreeTest::TreeTestConcurrentGet_Post, this); -// CreateUnit("TreeTestBalance"). -// Pre(&TreeTest::TreeTestBalance_Pre, this). -// Add(&TreeTest::TreeTestBalance_ThreadMethod, this, -// NUM_TEST_THREADS, 1). -// Post(&TreeTest::TreeTestBalance_Post, this); + CreateUnit("TreeTestBalance"). + Pre(&TreeTest::TreeTestBalance_Pre, this). + Add(&TreeTest::TreeTestBalance_ThreadMethod, this, + NUM_TEST_THREADS, 1). + Post(&TreeTest::TreeTestBalance_Post, this); } template -- libgit2 0.26.0