Skip to content

Commit f61a84a

Browse files
committed
BTree: restore 64 bit initialization speed through fieldial distancing
1 parent ee88f46 commit f61a84a

File tree

1 file changed

+34
-26
lines changed
  • library/alloc/src/collections/btree

1 file changed

+34
-26
lines changed

library/alloc/src/collections/btree/node.rs

Lines changed: 34 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@
1010
// keys: [K; 2 * B - 1],
1111
// vals: [V; 2 * B - 1],
1212
// edges: [if height > 0 { Box<Node<K, V, height - 1>> } else { () }; 2 * B],
13-
// parent: Option<(NonNull<Node<K, V, height + 1>>, u16)>,
14-
// len: u16,
13+
// parent: Option<(NonNull<Node<K, V, height + 1>>, PackedLength)>,
14+
// len: PackedLength,
1515
// }
1616
// ```
1717
//
@@ -47,6 +47,15 @@ const KV_IDX_CENTER: usize = B - 1;
4747
const EDGE_IDX_LEFT_OF_CENTER: usize = B - 1;
4848
const EDGE_IDX_RIGHT_OF_CENTER: usize = B;
4949

50+
#[cfg(target_pointer_width = "64")]
51+
type PackedLength = u32;
52+
53+
#[cfg(target_pointer_width = "32")]
54+
type PackedLength = u16;
55+
56+
#[cfg(target_pointer_width = "16")]
57+
type PackedLength = u8;
58+
5059
/// The underlying representation of leaf nodes and part of the representation of internal nodes.
5160
struct LeafNode<K, V> {
5261
/// We want to be covariant in `K` and `V`.
@@ -55,10 +64,10 @@ struct LeafNode<K, V> {
5564
/// This node's index into the parent node's `edges` array.
5665
/// `*node.parent.edges[node.parent_idx]` should be the same thing as `node`.
5766
/// This is only guaranteed to be initialized when `parent` is non-null.
58-
parent_idx: MaybeUninit<u16>,
67+
parent_idx: MaybeUninit<PackedLength>,
5968

6069
/// The number of keys and values this node stores.
61-
len: u16,
70+
len: PackedLength,
6271

6372
/// The arrays storing the actual data of the node. Only the first `len` elements of each
6473
/// array are initialized and valid.
@@ -341,7 +350,7 @@ impl<BorrowType, K, V, Type> NodeRef<BorrowType, K, V, Type> {
341350
pub fn len(&self) -> usize {
342351
// Crucially, we only access the `len` field here. If BorrowType is marker::ValMut,
343352
// there might be outstanding mutable references to values that we must not invalidate.
344-
unsafe { usize::from((*Self::as_leaf_ptr(self)).len) }
353+
unsafe { (*Self::as_leaf_ptr(self)).len as usize }
345354
}
346355

347356
/// Returns the number of levels that the node and leaves are apart. Zero
@@ -390,7 +399,7 @@ impl<BorrowType: marker::BorrowType, K, V, Type> NodeRef<BorrowType, K, V, Type>
390399
.as_ref()
391400
.map(|parent| Handle {
392401
node: NodeRef::from_internal(*parent, self.height + 1),
393-
idx: unsafe { usize::from((*leaf_ptr).parent_idx.assume_init()) },
402+
idx: unsafe { (*leaf_ptr).parent_idx.assume_init() as usize },
394403
_marker: PhantomData,
395404
})
396405
.ok_or(self)
@@ -431,9 +440,8 @@ impl<'a, K: 'a, V: 'a, Type> NodeRef<marker::Immut<'a>, K, V, Type> {
431440
/// Borrows a view into the keys stored in the node.
432441
pub fn keys(&self) -> &[K] {
433442
let leaf = self.into_leaf();
434-
unsafe {
435-
MaybeUninit::slice_assume_init_ref(leaf.keys.get_unchecked(..usize::from(leaf.len)))
436-
}
443+
let len = leaf.len as usize;
444+
unsafe { MaybeUninit::slice_assume_init_ref(leaf.keys.get_unchecked(..len)) }
437445
}
438446
}
439447

@@ -570,7 +578,7 @@ impl<'a, K, V, Type> NodeRef<marker::ValMut<'a>, K, V, Type> {
570578

571579
impl<'a, K: 'a, V: 'a, Type> NodeRef<marker::Mut<'a>, K, V, Type> {
572580
/// Borrows exclusive access to the length of the node.
573-
pub fn len_mut(&mut self) -> &mut u16 {
581+
pub fn len_mut(&mut self) -> &mut PackedLength {
574582
&mut self.as_leaf_mut().len
575583
}
576584
}
@@ -581,7 +589,7 @@ impl<'a, K: 'a, V: 'a> NodeRef<marker::Mut<'a>, K, V, marker::LeafOrInternal> {
581589
fn set_parent_link(&mut self, parent: NonNull<InternalNode<K, V>>, parent_idx: usize) {
582590
let leaf = Self::as_leaf_ptr(self);
583591
unsafe { (*leaf).parent = Some(parent) };
584-
unsafe { (*leaf).parent_idx.write(parent_idx as u16) };
592+
unsafe { (*leaf).parent_idx.write(parent_idx as PackedLength) };
585593
}
586594
}
587595

@@ -598,7 +606,7 @@ impl<'a, K: 'a, V: 'a> NodeRef<marker::Mut<'a>, K, V, marker::Leaf> {
598606
/// Adds a key-value pair to the end of the node.
599607
pub fn push(&mut self, key: K, val: V) {
600608
let len = self.len_mut();
601-
let idx = usize::from(*len);
609+
let idx = *len as usize;
602610
assert!(idx < CAPACITY);
603611
*len += 1;
604612
unsafe {
@@ -631,7 +639,7 @@ impl<'a, K: 'a, V: 'a> NodeRef<marker::Mut<'a>, K, V, marker::Internal> {
631639
assert!(edge.height == self.height - 1);
632640

633641
let len = self.len_mut();
634-
let idx = usize::from(*len);
642+
let idx = *len as usize;
635643
assert!(idx < CAPACITY);
636644
*len += 1;
637645
unsafe {
@@ -843,7 +851,7 @@ impl<'a, K: 'a, V: 'a> Handle<NodeRef<marker::Mut<'a>, K, V, marker::Leaf>, mark
843851
unsafe {
844852
slice_insert(self.node.key_area_mut(..new_len), self.idx, key);
845853
slice_insert(self.node.val_area_mut(..new_len), self.idx, val);
846-
*self.node.len_mut() = new_len as u16;
854+
*self.node.len_mut() = new_len as PackedLength;
847855

848856
self.node.val_area_mut(self.idx).assume_init_mut()
849857
}
@@ -903,7 +911,7 @@ impl<'a, K: 'a, V: 'a> Handle<NodeRef<marker::Mut<'a>, K, V, marker::Internal>,
903911
slice_insert(self.node.key_area_mut(..new_len), self.idx, key);
904912
slice_insert(self.node.val_area_mut(..new_len), self.idx, val);
905913
slice_insert(self.node.edge_area_mut(..new_len + 1), self.idx + 1, edge.node);
906-
*self.node.len_mut() = new_len as u16;
914+
*self.node.len_mut() = new_len as PackedLength;
907915

908916
self.node.correct_childrens_parent_links(self.idx + 1..new_len + 1);
909917
}
@@ -1057,7 +1065,7 @@ impl<'a, K: 'a, V: 'a, NodeType> Handle<NodeRef<marker::Mut<'a>, K, V, NodeType>
10571065
debug_assert!(self.idx < self.node.len());
10581066
let old_len = self.node.len();
10591067
let new_len = old_len - self.idx - 1;
1060-
new_node.len = new_len as u16;
1068+
new_node.len = new_len as PackedLength;
10611069
unsafe {
10621070
let k = self.node.key_area_mut(self.idx).assume_init_read();
10631071
let v = self.node.val_area_mut(self.idx).assume_init_read();
@@ -1071,7 +1079,7 @@ impl<'a, K: 'a, V: 'a, NodeType> Handle<NodeRef<marker::Mut<'a>, K, V, NodeType>
10711079
&mut new_node.vals[..new_len],
10721080
);
10731081

1074-
*self.node.len_mut() = self.idx as u16;
1082+
*self.node.len_mut() = self.idx as PackedLength;
10751083
(k, v)
10761084
}
10771085
}
@@ -1105,7 +1113,7 @@ impl<'a, K: 'a, V: 'a> Handle<NodeRef<marker::Mut<'a>, K, V, marker::Leaf>, mark
11051113
unsafe {
11061114
let k = slice_remove(self.node.key_area_mut(..old_len), self.idx);
11071115
let v = slice_remove(self.node.val_area_mut(..old_len), self.idx);
1108-
*self.node.len_mut() = (old_len - 1) as u16;
1116+
*self.node.len_mut() = (old_len - 1) as PackedLength;
11091117
((k, v), self.left_edge())
11101118
}
11111119
}
@@ -1124,7 +1132,7 @@ impl<'a, K: 'a, V: 'a> Handle<NodeRef<marker::Mut<'a>, K, V, marker::Internal>,
11241132
unsafe {
11251133
let mut new_node = InternalNode::new();
11261134
let kv = self.split_leaf_data(&mut new_node.data);
1127-
let new_len = usize::from(new_node.data.len);
1135+
let new_len = new_node.data.len as usize;
11281136
move_to_slice(
11291137
self.node.edge_area_mut(self.idx + 1..old_len + 1),
11301138
&mut new_node.edges[..new_len + 1],
@@ -1242,7 +1250,7 @@ impl<'a, K: 'a, V: 'a> BalancingContext<'a, K, V> {
12421250
assert!(new_left_len <= CAPACITY);
12431251

12441252
unsafe {
1245-
*left_node.len_mut() = new_left_len as u16;
1253+
*left_node.len_mut() = new_left_len as PackedLength;
12461254

12471255
let parent_key = slice_remove(parent_node.key_area_mut(..old_parent_len), parent_idx);
12481256
left_node.key_area_mut(old_left_len).write(parent_key);
@@ -1360,8 +1368,8 @@ impl<'a, K: 'a, V: 'a> BalancingContext<'a, K, V> {
13601368

13611369
let new_left_len = old_left_len - count;
13621370
let new_right_len = old_right_len + count;
1363-
*left_node.len_mut() = new_left_len as u16;
1364-
*right_node.len_mut() = new_right_len as u16;
1371+
*left_node.len_mut() = new_left_len as PackedLength;
1372+
*right_node.len_mut() = new_right_len as PackedLength;
13651373

13661374
// Move leaf data.
13671375
{
@@ -1423,8 +1431,8 @@ impl<'a, K: 'a, V: 'a> BalancingContext<'a, K, V> {
14231431

14241432
let new_left_len = old_left_len + count;
14251433
let new_right_len = old_right_len - count;
1426-
*left_node.len_mut() = new_left_len as u16;
1427-
*right_node.len_mut() = new_right_len as u16;
1434+
*left_node.len_mut() = new_left_len as PackedLength;
1435+
*right_node.len_mut() = new_right_len as PackedLength;
14281436

14291437
// Move leaf data.
14301438
{
@@ -1559,8 +1567,8 @@ impl<'a, K, V> Handle<NodeRef<marker::Mut<'a>, K, V, marker::LeafOrInternal>, ma
15591567
assert!(left_node.height == right_node.height);
15601568

15611569
if new_right_len > 0 {
1562-
*left_node.len_mut() = new_left_len as u16;
1563-
*right_node.len_mut() = new_right_len as u16;
1570+
*left_node.len_mut() = new_left_len as PackedLength;
1571+
*right_node.len_mut() = new_right_len as PackedLength;
15641572

15651573
move_to_slice(
15661574
left_node.key_area_mut(new_left_len..old_left_len),

0 commit comments

Comments
 (0)