Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit de88bf1

Browse files
Aaron1011Aaron1011
committed
Properly handle attributes on statements
We now collect tokens for the underlying node wrapped by `StmtKind` instead of storing tokens directly in `Stmt`. `LazyTokenStream` now supports capturing a trailing semicolon after it is initially constructed. This allows us to avoid refactoring statement parsing to wrap the parsing of the semicolon in `parse_tokens`. Attributes on item statements (e.g. `fn foo() { #[bar] struct MyStruct; }`) are now treated as item attributes, not statement attributes, which is consistent with how we handle attributes on other kinds of statements. The feature-gating code is adjusted so that proc-macro attributes are still allowed on item statements on stable. Two built-in macros (`#[global_allocator]` and `#[test]`) needed to be adjusted to support being passed `Annotatable::Stmt`.
1 parent 72da5a9 commit de88bf1

File tree

20 files changed

+484
-186
lines changed

20 files changed

+484
-186
lines changed

‎compiler/rustc_ast/src/ast.rs

Lines changed: 43 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -901,29 +901,65 @@ pub struct Stmt {
901901
pub id: NodeId,
902902
pub kind: StmtKind,
903903
pub span: Span,
904-
pub tokens: Option<LazyTokenStream>,
905904
}
906905

907906
impl Stmt {
907+
pub fn tokens(&self) -> Option<&LazyTokenStream> {
908+
match self.kind {
909+
StmtKind::Local(ref local) => local.tokens.as_ref(),
910+
StmtKind::Item(ref item) => item.tokens.as_ref(),
911+
StmtKind::Expr(ref expr) | StmtKind::Semi(ref expr) => expr.tokens.as_ref(),
912+
StmtKind::Empty => None,
913+
StmtKind::MacCall(ref mac) => mac.tokens.as_ref(),
914+
}
915+
}
916+
917+
pub fn tokens_mut(&mut self) -> Option<&mut LazyTokenStream> {
918+
match self.kind {
919+
StmtKind::Local(ref mut local) => local.tokens.as_mut(),
920+
StmtKind::Item(ref mut item) => item.tokens.as_mut(),
921+
StmtKind::Expr(ref mut expr) | StmtKind::Semi(ref mut expr) => expr.tokens.as_mut(),
922+
StmtKind::Empty => None,
923+
StmtKind::MacCall(ref mut mac) => mac.tokens.as_mut(),
924+
}
925+
}
926+
927+
pub fn set_tokens(&mut self, tokens: Option<LazyTokenStream>) {
928+
match self.kind {
929+
StmtKind::Local(ref mut local) => local.tokens = tokens,
930+
StmtKind::Item(ref mut item) => item.tokens = tokens,
931+
StmtKind::Expr(ref mut expr) | StmtKind::Semi(ref mut expr) => expr.tokens = tokens,
932+
StmtKind::Empty => {}
933+
StmtKind::MacCall(ref mut mac) => mac.tokens = tokens,
934+
}
935+
}
936+
908937
pub fn has_trailing_semicolon(&self) -> bool {
909938
match &self.kind {
910939
StmtKind::Semi(_) => true,
911940
StmtKind::MacCall(mac) => matches!(mac.style, MacStmtStyle::Semicolon),
912941
_ => false,
913942
}
914943
}
944+
945+
/// Converts a parsed `Stmt` to a `Stmt` with
946+
/// a trailing semicolon.
947+
///
948+
/// This only modifies the parsed AST struct, not the attached
949+
/// `LazyTokenStream`. The parser is responsible for calling
950+
/// `CreateTokenStream::add_trailing_semi` when there is actually
951+
/// a semicolon in the tokenstream.
915952
pub fn add_trailing_semicolon(mut self) -> Self {
916953
self.kind = match self.kind {
917954
StmtKind::Expr(expr) => StmtKind::Semi(expr),
918955
StmtKind::MacCall(mac) => {
919-
StmtKind::MacCall(mac.map(|MacCallStmt { mac, style: _, attrs }| MacCallStmt {
920-
mac,
921-
style: MacStmtStyle::Semicolon,
922-
attrs,
956+
StmtKind::MacCall(mac.map(|MacCallStmt { mac, style: _, attrs, tokens }| {
957+
MacCallStmt { mac, style: MacStmtStyle::Semicolon, attrs, tokens }
923958
}))
924959
}
925960
kind => kind,
926961
};
962+
927963
self
928964
}
929965

@@ -963,6 +999,7 @@ pub struct MacCallStmt {
963999
pub mac: MacCall,
9641000
pub style: MacStmtStyle,
9651001
pub attrs: AttrVec,
1002+
pub tokens: Option<LazyTokenStream>,
9661003
}
9671004

9681005
#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug)]
@@ -988,6 +1025,7 @@ pub struct Local {
9881025
pub init: Option<P<Expr>>,
9891026
pub span: Span,
9901027
pub attrs: AttrVec,
1028+
pub tokens: Option<LazyTokenStream>,
9911029
}
9921030

9931031
/// An arm of a 'match'.

‎compiler/rustc_ast/src/mut_visit.rs

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -576,13 +576,14 @@ pub fn noop_visit_parenthesized_parameter_data<T: MutVisitor>(
576576
}
577577

578578
pub fn noop_visit_local<T: MutVisitor>(local: &mut P<Local>, vis: &mut T) {
579-
let Local { id, pat, ty, init, span, attrs } = local.deref_mut();
579+
let Local { id, pat, ty, init, span, attrs, tokens } = local.deref_mut();
580580
vis.visit_id(id);
581581
vis.visit_pat(pat);
582582
visit_opt(ty, |ty| vis.visit_ty(ty));
583583
visit_opt(init, |init| vis.visit_expr(init));
584584
vis.visit_span(span);
585585
visit_thin_attrs(attrs, vis);
586+
visit_lazy_tts(tokens, vis);
586587
}
587588

588589
pub fn noop_visit_attribute<T: MutVisitor>(attr: &mut Attribute, vis: &mut T) {
@@ -1325,16 +1326,12 @@ pub fn noop_filter_map_expr<T: MutVisitor>(mut e: P<Expr>, vis: &mut T) -> Optio
13251326
}
13261327

13271328
pub fn noop_flat_map_stmt<T: MutVisitor>(
1328-
Stmt { kind, mut span, mut id, mut tokens }: Stmt,
1329+
Stmt { kind, mut span, mut id }: Stmt,
13291330
vis: &mut T,
13301331
) -> SmallVec<[Stmt; 1]> {
13311332
vis.visit_id(&mut id);
13321333
vis.visit_span(&mut span);
1333-
visit_lazy_tts(&mut tokens, vis);
1334-
noop_flat_map_stmt_kind(kind, vis)
1335-
.into_iter()
1336-
.map(|kind| Stmt { id, kind, span, tokens: tokens.clone() })
1337-
.collect()
1334+
noop_flat_map_stmt_kind(kind, vis).into_iter().map(|kind| Stmt { id, kind, span }).collect()
13381335
}
13391336

13401337
pub fn noop_flat_map_stmt_kind<T: MutVisitor>(
@@ -1351,9 +1348,10 @@ pub fn noop_flat_map_stmt_kind<T: MutVisitor>(
13511348
StmtKind::Semi(expr) => vis.filter_map_expr(expr).into_iter().map(StmtKind::Semi).collect(),
13521349
StmtKind::Empty => smallvec![StmtKind::Empty],
13531350
StmtKind::MacCall(mut mac) => {
1354-
let MacCallStmt { mac: mac_, style: _, attrs } = mac.deref_mut();
1351+
let MacCallStmt { mac: mac_, style: _, attrs, tokens } = mac.deref_mut();
13551352
vis.visit_mac_call(mac_);
13561353
visit_thin_attrs(attrs, vis);
1354+
visit_lazy_tts(tokens, vis);
13571355
smallvec![StmtKind::MacCall(mac)]
13581356
}
13591357
}

‎compiler/rustc_ast/src/tokenstream.rs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,10 +121,14 @@ where
121121
}
122122

123123
pub trait CreateTokenStream: sync::Send + sync::Sync {
124+
fn add_trailing_semi(&self) -> Box<dyn CreateTokenStream>;
124125
fn create_token_stream(&self) -> TokenStream;
125126
}
126127

127128
impl CreateTokenStream for TokenStream {
129+
fn add_trailing_semi(&self) -> Box<dyn CreateTokenStream> {
130+
panic!("Cannot call `add_trailing_semi` on a `TokenStream`!");
131+
}
128132
fn create_token_stream(&self) -> TokenStream {
129133
self.clone()
130134
}
@@ -141,6 +145,13 @@ impl LazyTokenStream {
141145
LazyTokenStream(Lrc::new(Box::new(inner)))
142146
}
143147

148+
/// Extends the captured stream by one token,
149+
/// which must be a trailing semicolon. This
150+
/// affects the `TokenStream` created by `make_tokenstream`.
151+
pub fn add_trailing_semi(&self) -> LazyTokenStream {
152+
LazyTokenStream(Lrc::new(self.0.add_trailing_semi()))
153+
}
154+
144155
pub fn create_token_stream(&self) -> TokenStream {
145156
self.0.create_token_stream()
146157
}

‎compiler/rustc_ast/src/visit.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -686,7 +686,7 @@ pub fn walk_stmt<'a, V: Visitor<'a>>(visitor: &mut V, statement: &'a Stmt) {
686686
StmtKind::Expr(ref expr) | StmtKind::Semi(ref expr) => visitor.visit_expr(expr),
687687
StmtKind::Empty => {}
688688
StmtKind::MacCall(ref mac) => {
689-
let MacCallStmt { ref mac, style: _, ref attrs } = **mac;
689+
let MacCallStmt { ref mac, style: _, ref attrs, tokens: _ } = **mac;
690690
visitor.visit_mac_call(mac);
691691
for attr in attrs.iter() {
692692
visitor.visit_attribute(attr);

‎compiler/rustc_builtin_macros/src/deriving/debug.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,7 @@ fn stmt_let_underscore(cx: &mut ExtCtxt<'_>, sp: Span, expr: P<ast::Expr>) -> as
132132
id: ast::DUMMY_NODE_ID,
133133
span: sp,
134134
attrs: ast::AttrVec::new(),
135+
tokens: None,
135136
});
136-
ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span: sp, tokens: None }
137+
ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span: sp }
137138
}

‎compiler/rustc_builtin_macros/src/deriving/mod.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,6 @@ impl MultiItemModifier for BuiltinDerive {
6464
id: ast::DUMMY_NODE_ID,
6565
kind: ast::StmtKind::Item(a.expect_item()),
6666
span,
67-
tokens: None,
6867
})));
6968
});
7069
} else {

‎compiler/rustc_expand/src/base.rs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -374,7 +374,6 @@ macro_rules! make_stmts_default {
374374
id: ast::DUMMY_NODE_ID,
375375
span: e.span,
376376
kind: ast::StmtKind::Expr(e),
377-
tokens: None
378377
}]
379378
})
380379
};
@@ -617,7 +616,6 @@ impl MacResult for DummyResult {
617616
id: ast::DUMMY_NODE_ID,
618617
kind: ast::StmtKind::Expr(DummyResult::raw_expr(self.span, self.is_error)),
619618
span: self.span,
620-
tokens: None
621619
}])
622620
}
623621

‎compiler/rustc_expand/src/build.rs

Lines changed: 6 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -140,12 +140,7 @@ impl<'a> ExtCtxt<'a> {
140140
}
141141

142142
pub fn stmt_expr(&self, expr: P<ast::Expr>) -> ast::Stmt {
143-
ast::Stmt {
144-
id: ast::DUMMY_NODE_ID,
145-
span: expr.span,
146-
kind: ast::StmtKind::Expr(expr),
147-
tokens: None,
148-
}
143+
ast::Stmt { id: ast::DUMMY_NODE_ID, span: expr.span, kind: ast::StmtKind::Expr(expr) }
149144
}
150145

151146
pub fn stmt_let(&self, sp: Span, mutbl: bool, ident: Ident, ex: P<ast::Expr>) -> ast::Stmt {
@@ -162,13 +157,9 @@ impl<'a> ExtCtxt<'a> {
162157
id: ast::DUMMY_NODE_ID,
163158
span: sp,
164159
attrs: AttrVec::new(),
165-
});
166-
ast::Stmt {
167-
id: ast::DUMMY_NODE_ID,
168-
kind: ast::StmtKind::Local(local),
169-
span: sp,
170160
tokens: None,
171-
}
161+
});
162+
ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span: sp }
172163
}
173164

174165
// Generates `let _: Type;`, which is usually used for type assertions.
@@ -180,17 +171,13 @@ impl<'a> ExtCtxt<'a> {
180171
id: ast::DUMMY_NODE_ID,
181172
span,
182173
attrs: AttrVec::new(),
174+
tokens: None,
183175
});
184-
ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span, tokens: None }
176+
ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span }
185177
}
186178

187179
pub fn stmt_item(&self, sp: Span, item: P<ast::Item>) -> ast::Stmt {
188-
ast::Stmt {
189-
id: ast::DUMMY_NODE_ID,
190-
kind: ast::StmtKind::Item(item),
191-
span: sp,
192-
tokens: None,
193-
}
180+
ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Item(item), span: sp }
194181
}
195182

196183
pub fn block_expr(&self, expr: P<ast::Expr>) -> P<ast::Block> {
@@ -200,7 +187,6 @@ impl<'a> ExtCtxt<'a> {
200187
id: ast::DUMMY_NODE_ID,
201188
span: expr.span,
202189
kind: ast::StmtKind::Expr(expr),
203-
tokens: None,
204190
}],
205191
)
206192
}

‎compiler/rustc_expand/src/expand.rs

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1274,12 +1274,6 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
12741274
// we'll expand attributes on expressions separately
12751275
if !stmt.is_expr() {
12761276
let attr = if stmt.is_item() {
1277-
// FIXME: Implement proper token collection for statements
1278-
if let StmtKind::Item(item) = &mut stmt.kind {
1279-
stmt.tokens = item.tokens.take()
1280-
} else {
1281-
unreachable!()
1282-
};
12831277
self.take_first_attr(&mut stmt)
12841278
} else {
12851279
// Ignore derives on non-item statements for backwards compatibility.
@@ -1295,7 +1289,7 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
12951289
}
12961290

12971291
if let StmtKind::MacCall(mac) = stmt.kind {
1298-
let MacCallStmt { mac, style, attrs } = mac.into_inner();
1292+
let MacCallStmt { mac, style, attrs, tokens: _ } = mac.into_inner();
12991293
self.check_attributes(&attrs);
13001294
let mut placeholder =
13011295
self.collect_bang(mac, stmt.span, AstFragmentKind::Stmts).make_stmts();
@@ -1312,10 +1306,10 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
13121306
}
13131307

13141308
// The placeholder expander gives ids to statements, so we avoid folding the id here.
1315-
let ast::Stmt { id, kind, span, tokens } = stmt;
1309+
let ast::Stmt { id, kind, span } = stmt;
13161310
noop_flat_map_stmt_kind(kind, self)
13171311
.into_iter()
1318-
.map(|kind| ast::Stmt { id, kind, span, tokens: tokens.clone() })
1312+
.map(|kind| ast::Stmt { id, kind, span })
13191313
.collect()
13201314
}
13211315

‎compiler/rustc_expand/src/placeholders.rs

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -104,8 +104,9 @@ pub fn placeholder(
104104
mac: mac_placeholder(),
105105
style: ast::MacStmtStyle::Braces,
106106
attrs: ast::AttrVec::new(),
107+
tokens: None,
107108
});
108-
ast::Stmt { id, span, kind: ast::StmtKind::MacCall(mac), tokens: None }
109+
ast::Stmt { id, span, kind: ast::StmtKind::MacCall(mac) }
109110
}]),
110111
AstFragmentKind::Arms => AstFragment::Arms(smallvec![ast::Arm {
111112
attrs: Default::default(),
@@ -331,12 +332,8 @@ impl<'a, 'b> MutVisitor for PlaceholderExpander<'a, 'b> {
331332

332333
// FIXME: We will need to preserve the original semicolon token and
333334
// span as part of #15701
334-
let empty_stmt = ast::Stmt {
335-
id: ast::DUMMY_NODE_ID,
336-
kind: ast::StmtKind::Empty,
337-
span: DUMMY_SP,
338-
tokens: None,
339-
};
335+
let empty_stmt =
336+
ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Empty, span: DUMMY_SP };
340337

341338
if let Some(stmt) = stmts.pop() {
342339
if stmt.has_trailing_semicolon() {

‎compiler/rustc_interface/src/util.rs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -810,7 +810,6 @@ impl<'a> MutVisitor for ReplaceBodyWithLoop<'a, '_> {
810810
id: resolver.next_node_id(),
811811
kind: ast::StmtKind::Expr(expr),
812812
span: rustc_span::DUMMY_SP,
813-
tokens: None,
814813
}
815814
}
816815

@@ -827,7 +826,6 @@ impl<'a> MutVisitor for ReplaceBodyWithLoop<'a, '_> {
827826
id: self.resolver.next_node_id(),
828827
span: rustc_span::DUMMY_SP,
829828
kind: ast::StmtKind::Expr(loop_expr),
830-
tokens: None,
831829
};
832830

833831
if self.within_static_or_const {

‎compiler/rustc_parse/src/lib.rs

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#![feature(or_patterns)]
77

88
use rustc_ast as ast;
9+
use rustc_ast::attr::HasAttrs;
910
use rustc_ast::token::{self, DelimToken, Nonterminal, Token, TokenKind};
1011
use rustc_ast::tokenstream::{self, LazyTokenStream, TokenStream, TokenTree};
1112
use rustc_ast_pretty::pprust;
@@ -249,29 +250,23 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke
249250
// before we fall back to the stringification.
250251

251252
let convert_tokens =
252-
|tokens: &Option<LazyTokenStream>| tokens.as_ref().map(|t| t.create_token_stream());
253+
|tokens: Option<&LazyTokenStream>| tokens.as_ref().map(|t| t.create_token_stream());
253254

254255
let tokens = match *nt {
255256
Nonterminal::NtItem(ref item) => prepend_attrs(&item.attrs, item.tokens.as_ref()),
256-
Nonterminal::NtBlock(ref block) => convert_tokens(&block.tokens),
257-
Nonterminal::NtStmt(ref stmt) => {
258-
// FIXME: We currently only collect tokens for `:stmt`
259-
// matchers in `macro_rules!` macros. When we start collecting
260-
// tokens for attributes on statements, we will need to prepend
261-
// attributes here
262-
convert_tokens(&stmt.tokens)
263-
}
264-
Nonterminal::NtPat(ref pat) => convert_tokens(&pat.tokens),
265-
Nonterminal::NtTy(ref ty) => convert_tokens(&ty.tokens),
257+
Nonterminal::NtBlock(ref block) => convert_tokens(block.tokens.as_ref()),
258+
Nonterminal::NtStmt(ref stmt) => prepend_attrs(stmt.attrs(), stmt.tokens()),
259+
Nonterminal::NtPat(ref pat) => convert_tokens(pat.tokens.as_ref()),
260+
Nonterminal::NtTy(ref ty) => convert_tokens(ty.tokens.as_ref()),
266261
Nonterminal::NtIdent(ident, is_raw) => {
267262
Some(tokenstream::TokenTree::token(token::Ident(ident.name, is_raw), ident.span).into())
268263
}
269264
Nonterminal::NtLifetime(ident) => {
270265
Some(tokenstream::TokenTree::token(token::Lifetime(ident.name), ident.span).into())
271266
}
272-
Nonterminal::NtMeta(ref attr) => convert_tokens(&attr.tokens),
273-
Nonterminal::NtPath(ref path) => convert_tokens(&path.tokens),
274-
Nonterminal::NtVis(ref vis) => convert_tokens(&vis.tokens),
267+
Nonterminal::NtMeta(ref attr) => convert_tokens(attr.tokens.as_ref()),
268+
Nonterminal::NtPath(ref path) => convert_tokens(path.tokens.as_ref()),
269+
Nonterminal::NtVis(ref vis) => convert_tokens(vis.tokens.as_ref()),
275270
Nonterminal::NtTT(ref tt) => Some(tt.clone().into()),
276271
Nonterminal::NtExpr(ref expr) | Nonterminal::NtLiteral(ref expr) => {
277272
if expr.tokens.is_none() {

0 commit comments

Comments
 (0)