Skip to content

Commit 7fb639f

Browse files
borsbors
committed
Auto merge of #16482 - GnomedDev:boxed-subtree, r=lnicola
Swap Subtree::token_trees from Vec to boxed slice Performs one of the optimizations suggested in #16325, but a little bit more. Boxed slices guarantee `shrink_to_fit` aswell as saving a pointer width as no capacity has to be stored. Most of the diff is: - Changing `vec![]` to `Box::new([])` - Changing initialize -> fill into fill -> into_boxed_slice - Working around the lack of an owned iterator or automatic iteration over a `Box<[T]>` I would like to use my own crate, [small-fixed-array](https://lib.rs/small-fixed-array), although I understand if it isn't mature enough for this. If I'm given the go ahead, I can rework this PR to use it instead.
2 parents 3e47532 + a607e1b commit 7fb639f

File tree

22 files changed

+196
-153
lines changed

22 files changed

+196
-153
lines changed

‎crates/hir-def/src/nameres/collector.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2125,7 +2125,7 @@ impl ModCollector<'_, '_> {
21252125

21262126
let is_export = export_attr.exists();
21272127
let local_inner = if is_export {
2128-
export_attr.tt_values().flat_map(|it| &it.token_trees).any(|it| match it {
2128+
export_attr.tt_values().flat_map(|it| it.token_trees.iter()).any(|it| match it {
21292129
tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => {
21302130
ident.text.contains("local_inner_macros")
21312131
}

‎crates/hir-expand/src/attrs.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ impl RawAttrs {
123123
.filter_map(|(idx, attr)| Attr::from_tt(db, attr, index.with_cfg_attr(idx)));
124124

125125
let cfg_options = &crate_graph[krate].cfg_options;
126-
let cfg = Subtree { delimiter: subtree.delimiter, token_trees: cfg.to_vec() };
126+
let cfg = Subtree { delimiter: subtree.delimiter, token_trees: Box::from(cfg) };
127127
let cfg = CfgExpr::parse(&cfg);
128128
if cfg_options.check(&cfg) == Some(false) {
129129
smallvec![]

‎crates/hir-expand/src/builtin_attr_macro.rs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,5 +137,8 @@ pub fn pseudo_derive_attr_expansion(
137137
token_trees.extend(tt.iter().cloned());
138138
token_trees.push(mk_leaf(']'));
139139
}
140-
ExpandResult::ok(tt::Subtree { delimiter: tt.delimiter, token_trees })
140+
ExpandResult::ok(tt::Subtree {
141+
delimiter: tt.delimiter,
142+
token_trees: token_trees.into_boxed_slice(),
143+
})
141144
}

‎crates/hir-expand/src/builtin_fn_macro.rs

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -155,10 +155,10 @@ fn line_expand(
155155
// not incremental
156156
ExpandResult::ok(tt::Subtree {
157157
delimiter: tt::Delimiter::invisible_spanned(span),
158-
token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
158+
token_trees: Box::new([tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
159159
text: "0u32".into(),
160160
span,
161-
}))],
161+
}))]),
162162
})
163163
}
164164

@@ -208,11 +208,11 @@ fn assert_expand(
208208
[cond, panic_args @ ..] => {
209209
let comma = tt::Subtree {
210210
delimiter: tt::Delimiter::invisible_spanned(call_site_span),
211-
token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct {
211+
token_trees: Box::new([tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct {
212212
char: ',',
213213
spacing: tt::Spacing::Alone,
214214
span: call_site_span,
215-
}))],
215+
}))]),
216216
};
217217
let cond = cond.clone();
218218
let panic_args = itertools::Itertools::intersperse(panic_args.iter().cloned(), comma);
@@ -359,7 +359,10 @@ fn panic_expand(
359359
close: call_site_span,
360360
kind: tt::DelimiterKind::Parenthesis,
361361
};
362-
call.token_trees.push(tt::TokenTree::Subtree(subtree));
362+
363+
// FIXME(slow): quote! have a way to expand to builder to make this a vec!
364+
call.push(tt::TokenTree::Subtree(subtree));
365+
363366
ExpandResult::ok(call)
364367
}
365368

@@ -388,7 +391,10 @@ fn unreachable_expand(
388391
close: call_site_span,
389392
kind: tt::DelimiterKind::Parenthesis,
390393
};
391-
call.token_trees.push(tt::TokenTree::Subtree(subtree));
394+
395+
// FIXME(slow): quote! have a way to expand to builder to make this a vec!
396+
call.push(tt::TokenTree::Subtree(subtree));
397+
392398
ExpandResult::ok(call)
393399
}
394400

@@ -675,10 +681,10 @@ fn include_bytes_expand(
675681
// FIXME: actually read the file here if the user asked for macro expansion
676682
let res = tt::Subtree {
677683
delimiter: tt::Delimiter::invisible_spanned(span),
678-
token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
684+
token_trees: Box::new([tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
679685
text: r#"b"""#.into(),
680686
span,
681-
}))],
687+
}))]),
682688
};
683689
ExpandResult::ok(res)
684690
}

‎crates/hir-expand/src/db.rs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -524,7 +524,7 @@ fn macro_expand(
524524
return ExpandResult {
525525
value: CowArc::Owned(tt::Subtree {
526526
delimiter: tt::Delimiter::invisible_spanned(loc.call_site),
527-
token_trees: Vec::new(),
527+
token_trees: Box::new([]),
528528
}),
529529
// FIXME: We should make sure to enforce an invariant that invalid macro
530530
// calls do not reach this call path!
@@ -586,7 +586,7 @@ fn macro_expand(
586586
return value.map(|()| {
587587
CowArc::Owned(tt::Subtree {
588588
delimiter: tt::Delimiter::invisible_spanned(loc.call_site),
589-
token_trees: vec![],
589+
token_trees: Box::new([]),
590590
})
591591
});
592592
}
@@ -601,7 +601,7 @@ fn expand_proc_macro(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult<A
601601
return ExpandResult {
602602
value: Arc::new(tt::Subtree {
603603
delimiter: tt::Delimiter::invisible_spanned(loc.call_site),
604-
token_trees: Vec::new(),
604+
token_trees: Box::new([]),
605605
}),
606606
// FIXME: We should make sure to enforce an invariant that invalid macro
607607
// calls do not reach this call path!
@@ -635,7 +635,7 @@ fn expand_proc_macro(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult<A
635635
return value.map(|()| {
636636
Arc::new(tt::Subtree {
637637
delimiter: tt::Delimiter::invisible_spanned(loc.call_site),
638-
token_trees: vec![],
638+
token_trees: Box::new([]),
639639
})
640640
});
641641
}

‎crates/hir-expand/src/fixup.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -312,7 +312,7 @@ pub(crate) fn reverse_fixups(tt: &mut Subtree, undo_info: &SyntaxFixupUndoInfo)
312312
}
313313

314314
fn reverse_fixups_(tt: &mut Subtree, undo_info: &[Subtree]) {
315-
let tts = std::mem::take(&mut tt.token_trees);
315+
let tts = std::mem::take(&mut tt.token_trees).into_vec();
316316
tt.token_trees = tts
317317
.into_iter()
318318
// delete all fake nodes
@@ -343,7 +343,7 @@ fn reverse_fixups_(tt: &mut Subtree, undo_info: &[Subtree]) {
343343
// we have a fake node here, we need to replace it again with the original
344344
let original = undo_info[u32::from(leaf.span().range.start()) as usize].clone();
345345
if original.delimiter.kind == tt::DelimiterKind::Invisible {
346-
original.token_trees.into()
346+
SmallVec::from(original.token_trees.into_vec())
347347
} else {
348348
SmallVec::from_const([original.into()])
349349
}
@@ -383,7 +383,7 @@ mod tests {
383383
fn check_subtree_eq(a: &tt::Subtree, b: &tt::Subtree) -> bool {
384384
a.delimiter.kind == b.delimiter.kind
385385
&& a.token_trees.len() == b.token_trees.len()
386-
&& a.token_trees.iter().zip(&b.token_trees).all(|(a, b)| check_tt_eq(a, b))
386+
&& a.token_trees.iter().zip(b.token_trees.iter()).all(|(a, b)| check_tt_eq(a, b))
387387
}
388388

389389
fn check_tt_eq(a: &tt::TokenTree, b: &tt::TokenTree) -> bool {

‎crates/hir-expand/src/lib.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ pub mod tt {
6666
pub type Delimiter = ::tt::Delimiter<Span>;
6767
pub type DelimSpan = ::tt::DelimSpan<Span>;
6868
pub type Subtree = ::tt::Subtree<Span>;
69+
pub type SubtreeBuilder = ::tt::SubtreeBuilder<Span>;
6970
pub type Leaf = ::tt::Leaf<Span>;
7071
pub type Literal = ::tt::Literal<Span>;
7172
pub type Punct = ::tt::Punct<Span>;
@@ -760,7 +761,7 @@ impl ExpansionInfo {
760761
(
761762
Arc::new(tt::Subtree {
762763
delimiter: tt::Delimiter::invisible_spanned(loc.call_site),
763-
token_trees: Vec::new(),
764+
token_trees: Box::new([]),
764765
}),
765766
SyntaxFixupUndoInfo::NONE,
766767
)

‎crates/hir-expand/src/quote.rs

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ macro_rules! __quote {
3131
open: $span,
3232
close: $span,
3333
},
34-
token_trees: $crate::quote::IntoTt::to_tokens(children),
34+
token_trees: $crate::quote::IntoTt::to_tokens(children).into_boxed_slice(),
3535
}
3636
}
3737
};
@@ -146,7 +146,7 @@ impl IntoTt for Vec<crate::tt::TokenTree> {
146146
fn to_subtree(self, span: Span) -> crate::tt::Subtree {
147147
crate::tt::Subtree {
148148
delimiter: crate::tt::Delimiter::invisible_spanned(span),
149-
token_trees: self,
149+
token_trees: self.into_boxed_slice(),
150150
}
151151
}
152152

@@ -296,8 +296,9 @@ mod tests {
296296
// }
297297
let struct_name = mk_ident("Foo");
298298
let fields = [mk_ident("name"), mk_ident("id")];
299-
let fields =
300-
fields.iter().flat_map(|it| quote!(DUMMY =>#it: self.#it.clone(), ).token_trees);
299+
let fields = fields
300+
.iter()
301+
.flat_map(|it| quote!(DUMMY =>#it: self.#it.clone(), ).token_trees.into_vec());
301302

302303
let list = crate::tt::Subtree {
303304
delimiter: crate::tt::Delimiter {

‎crates/mbe/src/benchmark.rs

Lines changed: 39 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -100,17 +100,19 @@ fn invocation_fixtures(
100100
// So we just skip any error cases and try again
101101
let mut try_cnt = 0;
102102
loop {
103-
let mut subtree = tt::Subtree {
103+
let mut token_trees = Vec::new();
104+
for op in rule.lhs.iter() {
105+
collect_from_op(op, &mut token_trees, &mut seed);
106+
}
107+
108+
let subtree = tt::Subtree {
104109
delimiter: tt::Delimiter {
105110
open: DUMMY,
106111
close: DUMMY,
107112
kind: tt::DelimiterKind::Invisible,
108113
},
109-
token_trees: vec![],
114+
token_trees: token_trees.into_boxed_slice(),
110115
};
111-
for op in rule.lhs.iter() {
112-
collect_from_op(op, &mut subtree, &mut seed);
113-
}
114116
if it.expand(&subtree, |_| (), true, DUMMY).err.is_none() {
115117
res.push((name.clone(), subtree));
116118
break;
@@ -127,45 +129,45 @@ fn invocation_fixtures(
127129

128130
fn collect_from_op(
129131
op: &Op<DummyTestSpanData>,
130-
parent: &mut tt::Subtree<DummyTestSpanData>,
132+
token_trees: &mut Vec<tt::TokenTree<DummyTestSpanData>>,
131133
seed: &mut usize,
132134
) {
133135
return match op {
134136
Op::Var { kind, .. } => match kind.as_ref() {
135-
Some(MetaVarKind::Ident) => parent.token_trees.push(make_ident("foo")),
136-
Some(MetaVarKind::Ty) => parent.token_trees.push(make_ident("Foo")),
137-
Some(MetaVarKind::Tt) => parent.token_trees.push(make_ident("foo")),
138-
Some(MetaVarKind::Vis) => parent.token_trees.push(make_ident("pub")),
139-
Some(MetaVarKind::Pat) => parent.token_trees.push(make_ident("foo")),
140-
Some(MetaVarKind::Path) => parent.token_trees.push(make_ident("foo")),
141-
Some(MetaVarKind::Literal) => parent.token_trees.push(make_literal("1")),
142-
Some(MetaVarKind::Expr) => parent.token_trees.push(make_ident("foo")),
137+
Some(MetaVarKind::Ident) => token_trees.push(make_ident("foo")),
138+
Some(MetaVarKind::Ty) => token_trees.push(make_ident("Foo")),
139+
Some(MetaVarKind::Tt) => token_trees.push(make_ident("foo")),
140+
Some(MetaVarKind::Vis) => token_trees.push(make_ident("pub")),
141+
Some(MetaVarKind::Pat) => token_trees.push(make_ident("foo")),
142+
Some(MetaVarKind::Path) => token_trees.push(make_ident("foo")),
143+
Some(MetaVarKind::Literal) => token_trees.push(make_literal("1")),
144+
Some(MetaVarKind::Expr) => token_trees.push(make_ident("foo")),
143145
Some(MetaVarKind::Lifetime) => {
144-
parent.token_trees.push(make_punct('\''));
145-
parent.token_trees.push(make_ident("a"));
146+
token_trees.push(make_punct('\''));
147+
token_trees.push(make_ident("a"));
146148
}
147149
Some(MetaVarKind::Block) => {
148-
parent.token_trees.push(make_subtree(tt::DelimiterKind::Brace, None))
150+
token_trees.push(make_subtree(tt::DelimiterKind::Brace, None))
149151
}
150152
Some(MetaVarKind::Item) => {
151-
parent.token_trees.push(make_ident("fn"));
152-
parent.token_trees.push(make_ident("foo"));
153-
parent.token_trees.push(make_subtree(tt::DelimiterKind::Parenthesis, None));
154-
parent.token_trees.push(make_subtree(tt::DelimiterKind::Brace, None));
153+
token_trees.push(make_ident("fn"));
154+
token_trees.push(make_ident("foo"));
155+
token_trees.push(make_subtree(tt::DelimiterKind::Parenthesis, None));
156+
token_trees.push(make_subtree(tt::DelimiterKind::Brace, None));
155157
}
156158
Some(MetaVarKind::Meta) => {
157-
parent.token_trees.push(make_ident("foo"));
158-
parent.token_trees.push(make_subtree(tt::DelimiterKind::Parenthesis, None));
159+
token_trees.push(make_ident("foo"));
160+
token_trees.push(make_subtree(tt::DelimiterKind::Parenthesis, None));
159161
}
160162

161163
None => (),
162164
Some(kind) => panic!("Unhandled kind {kind:?}"),
163165
},
164-
Op::Literal(it) => parent.token_trees.push(tt::Leaf::from(it.clone()).into()),
165-
Op::Ident(it) => parent.token_trees.push(tt::Leaf::from(it.clone()).into()),
166+
Op::Literal(it) => token_trees.push(tt::Leaf::from(it.clone()).into()),
167+
Op::Ident(it) => token_trees.push(tt::Leaf::from(it.clone()).into()),
166168
Op::Punct(puncts) => {
167169
for punct in puncts {
168-
parent.token_trees.push(tt::Leaf::from(*punct).into());
170+
token_trees.push(tt::Leaf::from(*punct).into());
169171
}
170172
}
171173
Op::Repeat { tokens, kind, separator } => {
@@ -177,20 +179,20 @@ fn invocation_fixtures(
177179
};
178180
for i in 0..cnt {
179181
for it in tokens.iter() {
180-
collect_from_op(it, parent, seed);
182+
collect_from_op(it, token_trees, seed);
181183
}
182184
if i + 1 != cnt {
183185
if let Some(sep) = separator {
184186
match sep {
185187
Separator::Literal(it) => {
186-
parent.token_trees.push(tt::Leaf::Literal(it.clone()).into())
188+
token_trees.push(tt::Leaf::Literal(it.clone()).into())
187189
}
188190
Separator::Ident(it) => {
189-
parent.token_trees.push(tt::Leaf::Ident(it.clone()).into())
191+
token_trees.push(tt::Leaf::Ident(it.clone()).into())
190192
}
191193
Separator::Puncts(puncts) => {
192194
for it in puncts {
193-
parent.token_trees.push(tt::Leaf::Punct(*it).into())
195+
token_trees.push(tt::Leaf::Punct(*it).into())
194196
}
195197
}
196198
};
@@ -199,11 +201,15 @@ fn invocation_fixtures(
199201
}
200202
}
201203
Op::Subtree { tokens, delimiter } => {
202-
let mut subtree = tt::Subtree { delimiter: *delimiter, token_trees: Vec::new() };
204+
let mut subtree = Vec::new();
203205
tokens.iter().for_each(|it| {
204206
collect_from_op(it, &mut subtree, seed);
205207
});
206-
parent.token_trees.push(subtree.into());
208+
209+
let subtree =
210+
tt::Subtree { delimiter: *delimiter, token_trees: subtree.into_boxed_slice() };
211+
212+
token_trees.push(subtree.into());
207213
}
208214
Op::Ignore { .. } | Op::Index { .. } | Op::Count { .. } | Op::Length { .. } => {}
209215
};
@@ -230,7 +236,7 @@ fn invocation_fixtures(
230236
) -> tt::TokenTree<DummyTestSpanData> {
231237
tt::Subtree {
232238
delimiter: tt::Delimiter { open: DUMMY, close: DUMMY, kind },
233-
token_trees: token_trees.unwrap_or_default(),
239+
token_trees: token_trees.map(Vec::into_boxed_slice).unwrap_or_default(),
234240
}
235241
.into()
236242
}

‎crates/mbe/src/expander.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ pub(crate) fn expand_rules<S: Span>(
5858
ExpandResult::new(
5959
tt::Subtree {
6060
delimiter: tt::Delimiter::invisible_spanned(call_site),
61-
token_trees: vec![],
61+
token_trees: Box::new([]),
6262
},
6363
ExpandError::NoMatchingRule,
6464
)

‎crates/mbe/src/expander/matcher.rs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -798,7 +798,7 @@ fn match_meta_var<S: Span>(
798798
tt.map(|tt| match tt {
799799
tt::TokenTree::Leaf(leaf) => tt::Subtree {
800800
delimiter: tt::Delimiter::invisible_spanned(*leaf.span()),
801-
token_trees: vec![leaf.into()],
801+
token_trees: Box::new([leaf.into()]),
802802
},
803803
tt::TokenTree::Subtree(mut s) => {
804804
if s.delimiter.kind == tt::DelimiterKind::Invisible {
@@ -832,7 +832,7 @@ fn match_meta_var<S: Span>(
832832
None => lit.into(),
833833
Some(neg) => tt::TokenTree::Subtree(tt::Subtree {
834834
delimiter: tt::Delimiter::invisible_spanned(*literal.span()),
835-
token_trees: vec![neg, lit.into()],
835+
token_trees: Box::new([neg, lit.into()]),
836836
}),
837837
}
838838
})
@@ -989,10 +989,10 @@ impl<S: Span> TtIter<'_, S> {
989989
close: ident.span,
990990
kind: tt::DelimiterKind::Invisible,
991991
},
992-
token_trees: vec![
992+
token_trees: Box::new([
993993
tt::Leaf::Punct(*punct).into(),
994994
tt::Leaf::Ident(ident.clone()).into(),
995-
],
995+
]),
996996
}
997997
.into())
998998
}

0 commit comments

Comments
 (0)