Skip to content

Commit 9c079ad

Browse files
walken-googletorvalds
walken-google
authored and
torvalds
committed
rbtree: move augmented rbtree functionality to rbtree_augmented.h
Provide rb_insert_augmented() and rb_erase_augmented() through a new rbtree_augmented.h include file. rb_erase_augmented() is defined there as an __always_inline function, in order to allow inlining of augmented rbtree callbacks into it. Since this generates a relatively large function, each augmented rbtree user should make sure to have a single call site. Signed-off-by: Michel Lespinasse <[email protected]> Cc: Rik van Riel <[email protected]> Cc: Hillf Danton <[email protected]> Cc: Peter Zijlstra <[email protected]> Cc: Catalin Marinas <[email protected]> Cc: Andrea Arcangeli <[email protected]> Cc: David Woodhouse <[email protected]> Signed-off-by: Andrew Morton <[email protected]> Signed-off-by: Linus Torvalds <[email protected]>
1 parent 147e615 commit 9c079ad

File tree

7 files changed

+255
-203
lines changed

7 files changed

+255
-203
lines changed

‎Documentation/rbtree.txt

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,14 @@ An rbtree user who wants this feature will have to call the augmentation
202202
functions with the user provided augmentation callback when inserting
203203
and erasing nodes.
204204

205+
C files implementing augmented rbtree manipulation must include
206+
<linux/rbtree_augmented.h> instead of <linus/rbtree.h>. Note that
207+
linux/rbtree_augmented.h exposes some rbtree implementations details
208+
you are not expected to rely on; please stick to the documented APIs
209+
there and do not include <linux/rbtree_augmented.h> from header files
210+
either so as to minimize chances of your users accidentally relying on
211+
such implementation details.
212+
205213
On insertion, the user must update the augmented information on the path
206214
leading to the inserted node, then call rb_link_node() as usual and
207215
rb_augment_inserted() instead of the usual rb_insert_color() call.
@@ -227,6 +235,11 @@ In both cases, the callbacks are provided through struct rb_augment_callbacks.
227235
subtree to a newly assigned subtree root AND recomputes the augmented
228236
information for the former subtree root.
229237

238+
The compiled code for rb_erase_augmented() may inline the propagation and
239+
copy callbacks, which results in a large function, so each augmented rbtree
240+
user should have a single rb_erase_augmented() call site in order to limit
241+
compiled code size.
242+
230243

231244
Sample usage:
232245

‎arch/x86/mm/pat_rbtree.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
#include <linux/debugfs.h>
1313
#include <linux/kernel.h>
1414
#include <linux/module.h>
15-
#include <linux/rbtree.h>
15+
#include <linux/rbtree_augmented.h>
1616
#include <linux/sched.h>
1717
#include <linux/gfp.h>
1818

‎include/linux/interval_tree_tmpl.h

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
include/linux/interval_tree_tmpl.h
2020
*/
2121

22+
#include <linux/rbtree_augmented.h>
23+
2224
/*
2325
* Template for implementing interval trees
2426
*
@@ -57,7 +59,8 @@ static inline ITTYPE IT(compute_subtree_last)(ITSTRUCT *node)
5759
return max;
5860
}
5961

60-
static void IT(augment_propagate)(struct rb_node *rb, struct rb_node *stop)
62+
static inline void
63+
IT(augment_propagate)(struct rb_node *rb, struct rb_node *stop)
6164
{
6265
while (rb != stop) {
6366
ITSTRUCT *node = rb_entry(rb, ITSTRUCT, ITRB);
@@ -69,7 +72,8 @@ static void IT(augment_propagate)(struct rb_node *rb, struct rb_node *stop)
6972
}
7073
}
7174

72-
static void IT(augment_copy)(struct rb_node *rb_old, struct rb_node *rb_new)
75+
static inline void
76+
IT(augment_copy)(struct rb_node *rb_old, struct rb_node *rb_new)
7377
{
7478
ITSTRUCT *old = rb_entry(rb_old, ITSTRUCT, ITRB);
7579
ITSTRUCT *new = rb_entry(rb_new, ITSTRUCT, ITRB);

‎include/linux/rbtree.h

Lines changed: 0 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -62,54 +62,6 @@ extern void rb_insert_color(struct rb_node *, struct rb_root *);
6262
extern void rb_erase(struct rb_node *, struct rb_root *);
6363

6464

65-
struct rb_augment_callbacks {
66-
void (*propagate)(struct rb_node *node, struct rb_node *stop);
67-
void (*copy)(struct rb_node *old, struct rb_node *new);
68-
void (*rotate)(struct rb_node *old, struct rb_node *new);
69-
};
70-
71-
extern void __rb_insert_augmented(struct rb_node *node, struct rb_root *root,
72-
void (*augment_rotate)(struct rb_node *old, struct rb_node *new));
73-
extern void rb_erase_augmented(struct rb_node *node, struct rb_root *root,
74-
const struct rb_augment_callbacks *augment);
75-
static inline void
76-
rb_insert_augmented(struct rb_node *node, struct rb_root *root,
77-
const struct rb_augment_callbacks *augment)
78-
{
79-
__rb_insert_augmented(node, root, augment->rotate);
80-
}
81-
82-
#define RB_DECLARE_CALLBACKS(rbstatic, rbname, rbstruct, rbfield, \
83-
rbtype, rbaugmented, rbcompute) \
84-
static void rbname ## _propagate(struct rb_node *rb, struct rb_node *stop) \
85-
{ \
86-
while (rb != stop) { \
87-
rbstruct *node = rb_entry(rb, rbstruct, rbfield); \
88-
rbtype augmented = rbcompute(node); \
89-
if (node->rbaugmented == augmented) \
90-
break; \
91-
node->rbaugmented = augmented; \
92-
rb = rb_parent(&node->rbfield); \
93-
} \
94-
} \
95-
static void rbname ## _copy(struct rb_node *rb_old, struct rb_node *rb_new) \
96-
{ \
97-
rbstruct *old = rb_entry(rb_old, rbstruct, rbfield); \
98-
rbstruct *new = rb_entry(rb_new, rbstruct, rbfield); \
99-
new->rbaugmented = old->rbaugmented; \
100-
} \
101-
static void rbname ## _rotate(struct rb_node *rb_old, struct rb_node *rb_new) \
102-
{ \
103-
rbstruct *old = rb_entry(rb_old, rbstruct, rbfield); \
104-
rbstruct *new = rb_entry(rb_new, rbstruct, rbfield); \
105-
new->rbaugmented = old->rbaugmented; \
106-
old->rbaugmented = rbcompute(old); \
107-
} \
108-
rbstatic const struct rb_augment_callbacks rbname = { \
109-
rbname ## _propagate, rbname ## _copy, rbname ## _rotate \
110-
};
111-
112-
11365
/* Find logical next and previous nodes in a tree */
11466
extern struct rb_node *rb_next(const struct rb_node *);
11567
extern struct rb_node *rb_prev(const struct rb_node *);

‎include/linux/rbtree_augmented.h

Lines changed: 223 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,223 @@
1+
/*
2+
Red Black Trees
3+
(C) 1999 Andrea Arcangeli <[email protected]>
4+
(C) 2002 David Woodhouse <[email protected]>
5+
(C) 2012 Michel Lespinasse <[email protected]>
6+
7+
This program is free software; you can redistribute it and/or modify
8+
it under the terms of the GNU General Public License as published by
9+
the Free Software Foundation; either version 2 of the License, or
10+
(at your option) any later version.
11+
12+
This program is distributed in the hope that it will be useful,
13+
but WITHOUT ANY WARRANTY; without even the implied warranty of
14+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15+
GNU General Public License for more details.
16+
17+
You should have received a copy of the GNU General Public License
18+
along with this program; if not, write to the Free Software
19+
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20+
21+
linux/include/linux/rbtree_augmented.h
22+
*/
23+
24+
#ifndef _LINUX_RBTREE_AUGMENTED_H
25+
#define _LINUX_RBTREE_AUGMENTED_H
26+
27+
#include <linux/rbtree.h>
28+
29+
/*
30+
* Please note - only struct rb_augment_callbacks and the prototypes for
31+
* rb_insert_augmented() and rb_erase_augmented() are intended to be public.
32+
* The rest are implementation details you are not expected to depend on.
33+
*
34+
* See Documentation/rbtree.txt for documentation and samples.
35+
*/
36+
37+
struct rb_augment_callbacks {
38+
void (*propagate)(struct rb_node *node, struct rb_node *stop);
39+
void (*copy)(struct rb_node *old, struct rb_node *new);
40+
void (*rotate)(struct rb_node *old, struct rb_node *new);
41+
};
42+
43+
extern void __rb_insert_augmented(struct rb_node *node, struct rb_root *root,
44+
void (*augment_rotate)(struct rb_node *old, struct rb_node *new));
45+
static inline void
46+
rb_insert_augmented(struct rb_node *node, struct rb_root *root,
47+
const struct rb_augment_callbacks *augment)
48+
{
49+
__rb_insert_augmented(node, root, augment->rotate);
50+
}
51+
52+
#define RB_DECLARE_CALLBACKS(rbstatic, rbname, rbstruct, rbfield, \
53+
rbtype, rbaugmented, rbcompute) \
54+
static inline void \
55+
rbname ## _propagate(struct rb_node *rb, struct rb_node *stop) \
56+
{ \
57+
while (rb != stop) { \
58+
rbstruct *node = rb_entry(rb, rbstruct, rbfield); \
59+
rbtype augmented = rbcompute(node); \
60+
if (node->rbaugmented == augmented) \
61+
break; \
62+
node->rbaugmented = augmented; \
63+
rb = rb_parent(&node->rbfield); \
64+
} \
65+
} \
66+
static inline void \
67+
rbname ## _copy(struct rb_node *rb_old, struct rb_node *rb_new) \
68+
{ \
69+
rbstruct *old = rb_entry(rb_old, rbstruct, rbfield); \
70+
rbstruct *new = rb_entry(rb_new, rbstruct, rbfield); \
71+
new->rbaugmented = old->rbaugmented; \
72+
} \
73+
static void \
74+
rbname ## _rotate(struct rb_node *rb_old, struct rb_node *rb_new) \
75+
{ \
76+
rbstruct *old = rb_entry(rb_old, rbstruct, rbfield); \
77+
rbstruct *new = rb_entry(rb_new, rbstruct, rbfield); \
78+
new->rbaugmented = old->rbaugmented; \
79+
old->rbaugmented = rbcompute(old); \
80+
} \
81+
rbstatic const struct rb_augment_callbacks rbname = { \
82+
rbname ## _propagate, rbname ## _copy, rbname ## _rotate \
83+
};
84+
85+
86+
#define RB_RED 0
87+
#define RB_BLACK 1
88+
89+
#define __rb_parent(pc) ((struct rb_node *)(pc & ~3))
90+
91+
#define __rb_color(pc) ((pc) & 1)
92+
#define __rb_is_black(pc) __rb_color(pc)
93+
#define __rb_is_red(pc) (!__rb_color(pc))
94+
#define rb_color(rb) __rb_color((rb)->__rb_parent_color)
95+
#define rb_is_red(rb) __rb_is_red((rb)->__rb_parent_color)
96+
#define rb_is_black(rb) __rb_is_black((rb)->__rb_parent_color)
97+
98+
static inline void rb_set_parent(struct rb_node *rb, struct rb_node *p)
99+
{
100+
rb->__rb_parent_color = rb_color(rb) | (unsigned long)p;
101+
}
102+
103+
static inline void rb_set_parent_color(struct rb_node *rb,
104+
struct rb_node *p, int color)
105+
{
106+
rb->__rb_parent_color = (unsigned long)p | color;
107+
}
108+
109+
static inline void
110+
__rb_change_child(struct rb_node *old, struct rb_node *new,
111+
struct rb_node *parent, struct rb_root *root)
112+
{
113+
if (parent) {
114+
if (parent->rb_left == old)
115+
parent->rb_left = new;
116+
else
117+
parent->rb_right = new;
118+
} else
119+
root->rb_node = new;
120+
}
121+
122+
extern void __rb_erase_color(struct rb_node *parent, struct rb_root *root,
123+
void (*augment_rotate)(struct rb_node *old, struct rb_node *new));
124+
125+
static __always_inline void
126+
rb_erase_augmented(struct rb_node *node, struct rb_root *root,
127+
const struct rb_augment_callbacks *augment)
128+
{
129+
struct rb_node *child = node->rb_right, *tmp = node->rb_left;
130+
struct rb_node *parent, *rebalance;
131+
unsigned long pc;
132+
133+
if (!tmp) {
134+
/*
135+
* Case 1: node to erase has no more than 1 child (easy!)
136+
*
137+
* Note that if there is one child it must be red due to 5)
138+
* and node must be black due to 4). We adjust colors locally
139+
* so as to bypass __rb_erase_color() later on.
140+
*/
141+
pc = node->__rb_parent_color;
142+
parent = __rb_parent(pc);
143+
__rb_change_child(node, child, parent, root);
144+
if (child) {
145+
child->__rb_parent_color = pc;
146+
rebalance = NULL;
147+
} else
148+
rebalance = __rb_is_black(pc) ? parent : NULL;
149+
tmp = parent;
150+
} else if (!child) {
151+
/* Still case 1, but this time the child is node->rb_left */
152+
tmp->__rb_parent_color = pc = node->__rb_parent_color;
153+
parent = __rb_parent(pc);
154+
__rb_change_child(node, tmp, parent, root);
155+
rebalance = NULL;
156+
tmp = parent;
157+
} else {
158+
struct rb_node *successor = child, *child2;
159+
tmp = child->rb_left;
160+
if (!tmp) {
161+
/*
162+
* Case 2: node's successor is its right child
163+
*
164+
* (n) (s)
165+
* / \ / \
166+
* (x) (s) -> (x) (c)
167+
* \
168+
* (c)
169+
*/
170+
parent = successor;
171+
child2 = successor->rb_right;
172+
augment->copy(node, successor);
173+
} else {
174+
/*
175+
* Case 3: node's successor is leftmost under
176+
* node's right child subtree
177+
*
178+
* (n) (s)
179+
* / \ / \
180+
* (x) (y) -> (x) (y)
181+
* / /
182+
* (p) (p)
183+
* / /
184+
* (s) (c)
185+
* \
186+
* (c)
187+
*/
188+
do {
189+
parent = successor;
190+
successor = tmp;
191+
tmp = tmp->rb_left;
192+
} while (tmp);
193+
parent->rb_left = child2 = successor->rb_right;
194+
successor->rb_right = child;
195+
rb_set_parent(child, successor);
196+
augment->copy(node, successor);
197+
augment->propagate(parent, successor);
198+
}
199+
200+
successor->rb_left = tmp = node->rb_left;
201+
rb_set_parent(tmp, successor);
202+
203+
pc = node->__rb_parent_color;
204+
tmp = __rb_parent(pc);
205+
__rb_change_child(node, successor, tmp, root);
206+
if (child2) {
207+
successor->__rb_parent_color = pc;
208+
rb_set_parent_color(child2, parent, RB_BLACK);
209+
rebalance = NULL;
210+
} else {
211+
unsigned long pc2 = successor->__rb_parent_color;
212+
successor->__rb_parent_color = pc;
213+
rebalance = __rb_is_black(pc2) ? parent : NULL;
214+
}
215+
tmp = successor;
216+
}
217+
218+
augment->propagate(tmp, NULL);
219+
if (rebalance)
220+
__rb_erase_color(rebalance, root, augment->rotate);
221+
}
222+
223+
#endif /* _LINUX_RBTREE_AUGMENTED_H */

0 commit comments

Comments
 (0)