|
33 | 33 |
|
34 | 34 | namespace swift {
|
35 | 35 |
|
36 |
| -/// This is a node in a concurrent linked list. |
37 |
| -template <class ElemTy> struct ConcurrentListNode { |
38 |
| - ConcurrentListNode(ElemTy Elem) : Payload(Elem), Next(nullptr) {} |
39 |
| - ConcurrentListNode(const ConcurrentListNode &) = delete; |
40 |
| - ConcurrentListNode &operator=(const ConcurrentListNode &) = delete; |
41 |
| - |
42 |
| - /// The element. |
43 |
| - ElemTy Payload; |
44 |
| - /// Points to the next link in the chain. |
45 |
| - ConcurrentListNode<ElemTy> *Next; |
46 |
| -}; |
47 |
| - |
48 |
| -/// This is a concurrent linked list. It supports insertion at the beginning |
49 |
| -/// of the list and traversal using iterators. |
50 |
| -/// This is a very simple implementation of a concurrent linked list |
51 |
| -/// using atomic operations. The 'push_front' method allocates a new link |
52 |
| -/// and attempts to compare and swap the old head pointer with pointer to |
53 |
| -/// the new link. This operation may fail many times if there are other |
54 |
| -/// contending threads, but eventually the head pointer is set to the new |
55 |
| -/// link that already points to the old head value. Notice that the more |
56 |
| -/// difficult feature of removing links is not supported. |
57 |
| -/// See 'push_front' for more details. |
58 |
| -template <class ElemTy> struct ConcurrentList { |
59 |
| - ConcurrentList() : First(nullptr) {} |
60 |
| - ~ConcurrentList() { |
61 |
| - clear(); |
62 |
| - } |
63 |
| - |
64 |
| - /// Remove all of the links in the chain. This method leaves |
65 |
| - /// the list at a usable state and new links can be added. |
66 |
| - /// Notice that this operation is non-sendable because |
67 |
| - /// we have no way of ensuring that no one is currently |
68 |
| - /// traversing the list. |
69 |
| - void clear() { |
70 |
| - // Iterate over the list and delete all the nodes. |
71 |
| - auto Ptr = First.load(std::memory_order_acquire); |
72 |
| - First.store(nullptr, std:: memory_order_release); |
73 |
| - |
74 |
| - while (Ptr) { |
75 |
| - auto N = Ptr->Next; |
76 |
| - delete Ptr; |
77 |
| - Ptr = N; |
78 |
| - } |
79 |
| - } |
80 |
| - |
81 |
| - ConcurrentList(const ConcurrentList &) = delete; |
82 |
| - ConcurrentList &operator=(const ConcurrentList &) = delete; |
83 |
| - |
84 |
| - /// A list iterator. |
85 |
| - struct ConcurrentListIterator : |
86 |
| - public std::iterator<std::forward_iterator_tag, ElemTy> { |
87 |
| - |
88 |
| - /// Points to the current link. |
89 |
| - ConcurrentListNode<ElemTy> *Ptr; |
90 |
| - /// C'tor. |
91 |
| - ConcurrentListIterator(ConcurrentListNode<ElemTy> *P) : Ptr(P) {} |
92 |
| - /// Move to the next element. |
93 |
| - ConcurrentListIterator &operator++() { |
94 |
| - Ptr = Ptr->Next; |
95 |
| - return *this; |
96 |
| - } |
97 |
| - /// Access the element. |
98 |
| - ElemTy &operator*() { return Ptr->Payload; } |
99 |
| - /// Same? |
100 |
| - bool operator==(const ConcurrentListIterator &o) const { |
101 |
| - return o.Ptr == Ptr; |
102 |
| - } |
103 |
| - /// Not the same? |
104 |
| - bool operator!=(const ConcurrentListIterator &o) const { |
105 |
| - return o.Ptr != Ptr; |
106 |
| - } |
107 |
| - }; |
108 |
| - |
109 |
| - /// Iterator entry point. |
110 |
| - typedef ConcurrentListIterator iterator; |
111 |
| - /// Marks the beginning of the list. |
112 |
| - iterator begin() const { |
113 |
| - return ConcurrentListIterator(First.load(std::memory_order_acquire)); |
114 |
| - } |
115 |
| - /// Marks the end of the list. |
116 |
| - iterator end() const { return ConcurrentListIterator(nullptr); } |
117 |
| - |
118 |
| - /// Add a new item to the list. |
119 |
| - void push_front(ElemTy Elem) { |
120 |
| - /// Allocate a new node. |
121 |
| - ConcurrentListNode<ElemTy> *N = new ConcurrentListNode<ElemTy>(Elem); |
122 |
| - // Point to the first element in the list. |
123 |
| - N->Next = First.load(std::memory_order_acquire); |
124 |
| - auto OldFirst = N->Next; |
125 |
| - // Try to replace the current First with the new node. |
126 |
| - while (!std::atomic_compare_exchange_weak_explicit(&First, &OldFirst, N, |
127 |
| - std::memory_order_release, |
128 |
| - std::memory_order_relaxed)) { |
129 |
| - // If we fail, update the new node to point to the new head and try to |
130 |
| - // insert before the new |
131 |
| - // first element. |
132 |
| - N->Next = OldFirst; |
133 |
| - } |
134 |
| - } |
135 |
| - |
136 |
| - /// Points to the first link in the list. |
137 |
| - std::atomic<ConcurrentListNode<ElemTy> *> First; |
138 |
| -}; |
139 |
| - |
140 |
| -/// A utility function for ordering two integers, which is useful |
141 |
| -/// for implementing compareWithKey. |
142 |
| -template <class T> |
143 |
| -static inline int compareIntegers(T left, T right) { |
144 |
| - return (left == right ? 0 : left < right ? -1 : 1); |
145 |
| -} |
146 |
| - |
147 |
| -/// A utility function for ordering two pointers, which is useful |
148 |
| -/// for implementing compareWithKey. |
149 |
| -template <class T> |
150 |
| -static inline int comparePointers(const T *left, const T *right) { |
151 |
| - return (left == right ? 0 : std::less<const T *>()(left, right) ? -1 : 1); |
152 |
| -} |
153 |
| - |
154 |
| -template <class EntryTy, bool ProvideDestructor, class Allocator> |
155 |
| -class ConcurrentMapBase; |
156 |
| - |
157 |
| -/// The partial specialization of ConcurrentMapBase whose destructor is |
158 |
| -/// trivial. The other implementation inherits from this, so this is a |
159 |
| -/// base for all ConcurrentMaps. |
160 |
| -template <class EntryTy, class Allocator> |
161 |
| -class ConcurrentMapBase<EntryTy, false, Allocator> : protected Allocator { |
162 |
| -protected: |
163 |
| - struct Node { |
164 |
| - std::atomic<Node*> Left; |
165 |
| - std::atomic<Node*> Right; |
166 |
| - EntryTy Payload; |
167 |
| - |
168 |
| - template <class... Args> |
169 |
| - Node(Args &&... args) |
170 |
| - : Left(nullptr), Right(nullptr), Payload(std::forward<Args>(args)...) {} |
171 |
| - |
172 |
| - Node(const Node &) = delete; |
173 |
| - Node &operator=(const Node &) = delete; |
174 |
| - |
175 |
| - #ifndef NDEBUG |
176 |
| - void dump() const { |
177 |
| - auto L = Left.load(std::memory_order_acquire); |
178 |
| - auto R = Right.load(std::memory_order_acquire); |
179 |
| - printf("\"%p\" [ label = \" {<f0> %08lx | {<f1> | <f2>}}\" " |
180 |
| - "style=\"rounded\" shape=\"record\"];\n", |
181 |
| - this, (long) Payload.getKeyValueForDump()); |
182 |
| - |
183 |
| - if (L) { |
184 |
| - L->dump(); |
185 |
| - printf("\"%p\":f1 -> \"%p\":f0;\n", this, L); |
186 |
| - } |
187 |
| - if (R) { |
188 |
| - R->dump(); |
189 |
| - printf("\"%p\":f2 -> \"%p\":f0;\n", this, R); |
190 |
| - } |
191 |
| - } |
192 |
| - #endif |
193 |
| - }; |
194 |
| - |
195 |
| - std::atomic<Node*> Root; |
196 |
| - |
197 |
| - constexpr ConcurrentMapBase() : Root(nullptr) {} |
198 |
| - |
199 |
| - // Implicitly trivial destructor. |
200 |
| - ~ConcurrentMapBase() = default; |
201 |
| - |
202 |
| - void destroyNode(Node *node) { |
203 |
| - assert(node && "destroying null node"); |
204 |
| - auto allocSize = sizeof(Node) + node->Payload.getExtraAllocationSize(); |
205 |
| - |
206 |
| - // Destroy the node's payload. |
207 |
| - node->~Node(); |
208 |
| - |
209 |
| - // Deallocate the node. The static_cast here is required |
210 |
| - // because LLVM's allocator API is insane. |
211 |
| - this->Deallocate(static_cast<void*>(node), allocSize, alignof(Node)); |
212 |
| - } |
213 |
| -}; |
214 |
| - |
215 |
| -/// The partial specialization of ConcurrentMapBase which provides a |
216 |
| -/// non-trivial destructor. |
217 |
| -template <class EntryTy, class Allocator> |
218 |
| -class ConcurrentMapBase<EntryTy, true, Allocator> |
219 |
| - : protected ConcurrentMapBase<EntryTy, false, Allocator> { |
220 |
| -protected: |
221 |
| - using super = ConcurrentMapBase<EntryTy, false, Allocator>; |
222 |
| - using Node = typename super::Node; |
223 |
| - |
224 |
| - constexpr ConcurrentMapBase() {} |
225 |
| - |
226 |
| - ~ConcurrentMapBase() { |
227 |
| - destroyTree(this->Root); |
228 |
| - } |
229 |
| - |
230 |
| -private: |
231 |
| - void destroyTree(const std::atomic<Node*> &edge) { |
232 |
| - // This can be a relaxed load because destruction is not allowed to race |
233 |
| - // with other operations. |
234 |
| - auto node = edge.load(std::memory_order_relaxed); |
235 |
| - if (!node) return; |
236 |
| - |
237 |
| - // Destroy the node's children. |
238 |
| - destroyTree(node->Left); |
239 |
| - destroyTree(node->Right); |
240 |
| - |
241 |
| - // Destroy the node itself. |
242 |
| - this->destroyNode(node); |
243 |
| - } |
244 |
| -}; |
245 |
| - |
246 |
| -/// A concurrent map that is implemented using a binary tree. It supports |
247 |
| -/// concurrent insertions but does not support removals or rebalancing of |
248 |
| -/// the tree. |
249 |
| -/// |
250 |
| -/// The entry type must provide the following operations: |
251 |
| -/// |
252 |
| -/// /// For debugging purposes only. Summarize this key as an integer value. |
253 |
| -/// intptr_t getKeyIntValueForDump() const; |
254 |
| -/// |
255 |
| -/// /// A ternary comparison. KeyTy is the type of the key provided |
256 |
| -/// /// to find or getOrInsert. |
257 |
| -/// int compareWithKey(KeyTy key) const; |
258 |
| -/// |
259 |
| -/// /// Return the amount of extra trailing space required by an entry, |
260 |
| -/// /// where KeyTy is the type of the first argument to getOrInsert and |
261 |
| -/// /// ArgTys is the type of the remaining arguments. |
262 |
| -/// static size_t getExtraAllocationSize(KeyTy key, ArgTys...) |
263 |
| -/// |
264 |
| -/// /// Return the amount of extra trailing space that was requested for |
265 |
| -/// /// this entry. This method is only used to compute the size of the |
266 |
| -/// /// object during node deallocation; it does not need to return a |
267 |
| -/// /// correct value so long as the allocator's Deallocate implementation |
268 |
| -/// /// ignores this argument. |
269 |
| -/// size_t getExtraAllocationSize() const; |
270 |
| -/// |
271 |
| -/// If ProvideDestructor is false, the destructor will be trivial. This |
272 |
| -/// can be appropriate when the object is declared at global scope. |
273 |
| -template <class EntryTy, bool ProvideDestructor = true, |
274 |
| - class Allocator = llvm::MallocAllocator> |
275 |
| -class ConcurrentMap |
276 |
| - : private ConcurrentMapBase<EntryTy, ProvideDestructor, Allocator> { |
277 |
| - using super = ConcurrentMapBase<EntryTy, ProvideDestructor, Allocator>; |
278 |
| - |
279 |
| - using Node = typename super::Node; |
280 |
| - |
281 |
| - /// Inherited from base class: |
282 |
| - /// std::atomic<Node*> Root; |
283 |
| - using super::Root; |
284 |
| - |
285 |
| - /// This member stores the address of the last node that was found by the |
286 |
| - /// search procedure. We cache the last search to accelerate code that |
287 |
| - /// searches the same value in a loop. |
288 |
| - std::atomic<Node*> LastSearch; |
289 |
| - |
290 |
| -public: |
291 |
| - constexpr ConcurrentMap() : LastSearch(nullptr) {} |
292 |
| - |
293 |
| - ConcurrentMap(const ConcurrentMap &) = delete; |
294 |
| - ConcurrentMap &operator=(const ConcurrentMap &) = delete; |
295 |
| - |
296 |
| - // ConcurrentMap<T, false> must have a trivial destructor. |
297 |
| - ~ConcurrentMap() = default; |
298 |
| - |
299 |
| -public: |
300 |
| - |
301 |
| - Allocator &getAllocator() { |
302 |
| - return *this; |
303 |
| - } |
304 |
| - |
305 |
| -#ifndef NDEBUG |
306 |
| - void dump() const { |
307 |
| - auto R = Root.load(std::memory_order_acquire); |
308 |
| - printf("digraph g {\n" |
309 |
| - "graph [ rankdir = \"TB\"];\n" |
310 |
| - "node [ fontsize = \"16\" ];\n" |
311 |
| - "edge [ ];\n"); |
312 |
| - if (R) { |
313 |
| - R->dump(); |
314 |
| - } |
315 |
| - printf("\n}\n"); |
316 |
| - } |
317 |
| -#endif |
318 |
| - |
319 |
| - /// Search for a value by key \p Key. |
320 |
| - /// \returns a pointer to the value or null if the value is not in the map. |
321 |
| - template <class KeyTy> |
322 |
| - EntryTy *find(const KeyTy &key) { |
323 |
| - // Check if we are looking for the same key that we looked for in the last |
324 |
| - // time we called this function. |
325 |
| - if (Node *last = LastSearch.load(std::memory_order_acquire)) { |
326 |
| - if (last->Payload.compareWithKey(key) == 0) |
327 |
| - return &last->Payload; |
328 |
| - } |
329 |
| - |
330 |
| - // Search the tree, starting from the root. |
331 |
| - Node *node = Root.load(std::memory_order_acquire); |
332 |
| - while (node) { |
333 |
| - int comparisonResult = node->Payload.compareWithKey(key); |
334 |
| - if (comparisonResult == 0) { |
335 |
| - LastSearch.store(node, std::memory_order_release); |
336 |
| - return &node->Payload; |
337 |
| - } else if (comparisonResult < 0) { |
338 |
| - node = node->Left.load(std::memory_order_acquire); |
339 |
| - } else { |
340 |
| - node = node->Right.load(std::memory_order_acquire); |
341 |
| - } |
342 |
| - } |
343 |
| - |
344 |
| - return nullptr; |
345 |
| - } |
346 |
| - |
347 |
| - /// Get or create an entry in the map. |
348 |
| - /// |
349 |
| - /// \returns the entry in the map and whether a new node was added (true) |
350 |
| - /// or already existed (false) |
351 |
| - template <class KeyTy, class... ArgTys> |
352 |
| - std::pair<EntryTy*, bool> getOrInsert(KeyTy key, ArgTys &&... args) { |
353 |
| - // Check if we are looking for the same key that we looked for the |
354 |
| - // last time we called this function. |
355 |
| - if (Node *last = LastSearch.load(std::memory_order_acquire)) { |
356 |
| - if (last && last->Payload.compareWithKey(key) == 0) |
357 |
| - return { &last->Payload, false }; |
358 |
| - } |
359 |
| - |
360 |
| - // The node we allocated. |
361 |
| - Node *newNode = nullptr; |
362 |
| - |
363 |
| - // Start from the root. |
364 |
| - auto edge = &Root; |
365 |
| - |
366 |
| - while (true) { |
367 |
| - // Load the edge. |
368 |
| - Node *node = edge->load(std::memory_order_acquire); |
369 |
| - |
370 |
| - // If there's a node there, it's either a match or we're going to |
371 |
| - // one of its children. |
372 |
| - if (node) { |
373 |
| - searchFromNode: |
374 |
| - |
375 |
| - // Compare our key against the node's key. |
376 |
| - int comparisonResult = node->Payload.compareWithKey(key); |
377 |
| - |
378 |
| - // If it's equal, we can use this node. |
379 |
| - if (comparisonResult == 0) { |
380 |
| - // Destroy the node we allocated before if we're carrying one around. |
381 |
| - if (newNode) this->destroyNode(newNode); |
382 |
| - |
383 |
| - // Cache and report that we found an existing node. |
384 |
| - LastSearch.store(node, std::memory_order_release); |
385 |
| - return { &node->Payload, false }; |
386 |
| - } |
387 |
| - |
388 |
| - // Otherwise, select the appropriate child edge and descend. |
389 |
| - edge = (comparisonResult < 0 ? &node->Left : &node->Right); |
390 |
| - continue; |
391 |
| - } |
392 |
| - |
393 |
| - // Create a new node. |
394 |
| - if (!newNode) { |
395 |
| - size_t allocSize = |
396 |
| - sizeof(Node) + EntryTy::getExtraAllocationSize(key, args...); |
397 |
| - void *memory = this->Allocate(allocSize, alignof(Node)); |
398 |
| - newNode = ::new (memory) Node(key, std::forward<ArgTys>(args)...); |
399 |
| - } |
400 |
| - |
401 |
| - // Try to set the edge to the new node. |
402 |
| - if (std::atomic_compare_exchange_strong_explicit(edge, &node, newNode, |
403 |
| - std::memory_order_acq_rel, |
404 |
| - std::memory_order_acquire)) { |
405 |
| - // If that succeeded, cache and report that we created a new node. |
406 |
| - LastSearch.store(newNode, std::memory_order_release); |
407 |
| - return { &newNode->Payload, true }; |
408 |
| - } |
409 |
| - |
410 |
| - // Otherwise, we lost the race because some other thread initialized |
411 |
| - // the edge before us. node will be set to the current value; |
412 |
| - // repeat the search from there. |
413 |
| - assert(node && "spurious failure from compare_exchange_strong?"); |
414 |
| - goto searchFromNode; |
415 |
| - } |
416 |
| - } |
417 |
| -}; |
418 |
| - |
419 | 36 | /// A simple linked list representing pointers that need to be freed. This is
|
420 | 37 | /// not a concurrent data structure, just a bit of support used in the types
|
421 | 38 | /// below.
|
|
0 commit comments