rustc_ast: Replace AstLike::finalize_tokens with a getter tokens_mut

petrochenkov · petrochenkov · commit 069e612e73f3 · 2021-03-06T21:19:31.000+03:00
diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs
@@ -915,16 +915,6 @@ impl Stmt {
         }
     }
 
-    pub fn tokens_mut(&mut self) -> Option<&mut LazyTokenStream> {
-        match self.kind {
-            StmtKind::Local(ref mut local) => local.tokens.as_mut(),
-            StmtKind::Item(ref mut item) => item.tokens.as_mut(),
-            StmtKind::Expr(ref mut expr) | StmtKind::Semi(ref mut expr) => expr.tokens.as_mut(),
-            StmtKind::Empty => None,
-            StmtKind::MacCall(ref mut mac) => mac.tokens.as_mut(),
-        }
-    }
-
     pub fn has_trailing_semicolon(&self) -> bool {
         match &self.kind {
             StmtKind::Semi(_) => true,
diff --git a/compiler/rustc_ast/src/ast_like.rs b/compiler/rustc_ast/src/ast_like.rs
@@ -11,13 +11,7 @@ use super::{AttrVec, Attribute, Stmt, StmtKind};
 pub trait AstLike: Sized {
     fn attrs(&self) -> &[Attribute];
     fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>));
-    /// Called by `Parser::collect_tokens` to store the collected
-    /// tokens inside an AST node
-    fn finalize_tokens(&mut self, _tokens: LazyTokenStream) {
-        // This default impl makes this trait easier to implement
-        // in tools like `rust-analyzer`
-        panic!("`finalize_tokens` is not supported!")
-    }
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>>;
 }
 
 impl<T: AstLike + 'static> AstLike for P<T> {
@@ -27,8 +21,8 @@ impl<T: AstLike + 'static> AstLike for P<T> {
     fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {
         (**self).visit_attrs(f);
     }
-    fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
-        (**self).finalize_tokens(tokens)
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+        (**self).tokens_mut()
     }
 }
 
@@ -42,12 +36,12 @@ fn visit_attrvec(attrs: &mut AttrVec, f: impl FnOnce(&mut Vec<Attribute>)) {
 
 impl AstLike for StmtKind {
     fn attrs(&self) -> &[Attribute] {
-        match *self {
-            StmtKind::Local(ref local) => local.attrs(),
-            StmtKind::Expr(ref expr) | StmtKind::Semi(ref expr) => expr.attrs(),
-            StmtKind::Item(ref item) => item.attrs(),
+        match self {
+            StmtKind::Local(local) => local.attrs(),
+            StmtKind::Expr(expr) | StmtKind::Semi(expr) => expr.attrs(),
+            StmtKind::Item(item) => item.attrs(),
             StmtKind::Empty => &[],
-            StmtKind::MacCall(ref mac) => &*mac.attrs,
+            StmtKind::MacCall(mac) => &mac.attrs,
         }
     }
 
@@ -60,17 +54,14 @@ impl AstLike for StmtKind {
             StmtKind::MacCall(mac) => visit_attrvec(&mut mac.attrs, f),
         }
     }
-    fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
-        let stmt_tokens = match self {
-            StmtKind::Local(ref mut local) => &mut local.tokens,
-            StmtKind::Item(ref mut item) => &mut item.tokens,
-            StmtKind::Expr(ref mut expr) | StmtKind::Semi(ref mut expr) => &mut expr.tokens,
-            StmtKind::Empty => return,
-            StmtKind::MacCall(ref mut mac) => &mut mac.tokens,
-        };
-        if stmt_tokens.is_none() {
-            *stmt_tokens = Some(tokens);
-        }
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+        Some(match self {
+            StmtKind::Local(local) => &mut local.tokens,
+            StmtKind::Item(item) => &mut item.tokens,
+            StmtKind::Expr(expr) | StmtKind::Semi(expr) => &mut expr.tokens,
+            StmtKind::Empty => return None,
+            StmtKind::MacCall(mac) => &mut mac.tokens,
+        })
     }
 }
 
@@ -82,8 +73,8 @@ impl AstLike for Stmt {
     fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {
         self.kind.visit_attrs(f);
     }
-    fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
-        self.kind.finalize_tokens(tokens)
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+        self.kind.tokens_mut()
     }
 }
 
@@ -92,17 +83,13 @@ impl AstLike for Attribute {
         &[]
     }
     fn visit_attrs(&mut self, _f: impl FnOnce(&mut Vec<Attribute>)) {}
-    fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
-        match &mut self.kind {
-            AttrKind::Normal(_, attr_tokens) => {
-                if attr_tokens.is_none() {
-                    *attr_tokens = Some(tokens);
-                }
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+        Some(match &mut self.kind {
+            AttrKind::Normal(_, tokens) => tokens,
+            kind @ AttrKind::DocComment(..) => {
+                panic!("Called tokens_mut on doc comment attr {:?}", kind)
             }
-            AttrKind::DocComment(..) => {
-                panic!("Called finalize_tokens on doc comment attr {:?}", self)
-            }
-        }
+        })
     }
 }
 
@@ -115,10 +102,8 @@ impl<T: AstLike> AstLike for Option<T> {
             inner.visit_attrs(f);
         }
     }
-    fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
-        if let Some(inner) = self {
-            inner.finalize_tokens(tokens);
-        }
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+        self.as_mut().and_then(|inner| inner.tokens_mut())
     }
 }
 
@@ -152,11 +137,8 @@ macro_rules! derive_has_tokens_and_attrs {
                 VecOrAttrVec::visit(&mut self.attrs, f)
             }
 
-            fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
-                if self.tokens.is_none() {
-                    self.tokens = Some(tokens);
-                }
-
+            fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+                Some(&mut self.tokens)
             }
         }
     )* }
@@ -173,7 +155,9 @@ macro_rules! derive_has_attrs_no_tokens {
                 VecOrAttrVec::visit(&mut self.attrs, f)
             }
 
-            fn finalize_tokens(&mut self, _tokens: LazyTokenStream) {}
+            fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+                None
+            }
         }
     )* }
 }
@@ -185,14 +169,10 @@ macro_rules! derive_has_tokens_no_attrs {
                 &[]
             }
 
-            fn visit_attrs(&mut self, _f: impl FnOnce(&mut Vec<Attribute>)) {
-            }
-
-            fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
-                if self.tokens.is_none() {
-                    self.tokens = Some(tokens);
-                }
+            fn visit_attrs(&mut self, _f: impl FnOnce(&mut Vec<Attribute>)) {}
 
+            fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+                Some(&mut self.tokens)
             }
         }
     )* }
diff --git a/compiler/rustc_builtin_macros/src/derive.rs b/compiler/rustc_builtin_macros/src/derive.rs
@@ -1,4 +1,4 @@
-use rustc_ast::{self as ast, token, ItemKind, MetaItemKind, NestedMetaItem, StmtKind};
+use rustc_ast::{self as ast, token, AstLike, ItemKind, MetaItemKind, NestedMetaItem, StmtKind};
 use rustc_errors::{struct_span_err, Applicability};
 use rustc_expand::base::{Annotatable, ExpandResult, ExtCtxt, Indeterminate, MultiItemModifier};
 use rustc_expand::config::StripUnconfigured;
@@ -59,15 +59,9 @@ impl MultiItemModifier for Expander {
                     // Erase the tokens if cfg-stripping modified the item
                     // This will cause us to synthesize fake tokens
                     // when `nt_to_tokenstream` is called on this item.
-                    match &mut item {
-                        Annotatable::Item(item) => item,
-                        Annotatable::Stmt(stmt) => match &mut stmt.kind {
-                            StmtKind::Item(item) => item,
-                            _ => unreachable!(),
-                        },
-                        _ => unreachable!(),
+                    if let Some(tokens) = item.tokens_mut() {
+                        *tokens = None;
                     }
-                    .tokens = None;
                 }
                 ExpandResult::Ready(vec![item])
             }
diff --git a/compiler/rustc_expand/src/base.rs b/compiler/rustc_expand/src/base.rs
@@ -81,8 +81,22 @@ impl AstLike for Annotatable {
         }
     }
 
-    fn finalize_tokens(&mut self, tokens: LazyTokenStream) {
-        panic!("Called finalize_tokens on an Annotatable: {:?}", tokens);
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+        match self {
+            Annotatable::Item(item) => item.tokens_mut(),
+            Annotatable::TraitItem(trait_item) => trait_item.tokens_mut(),
+            Annotatable::ImplItem(impl_item) => impl_item.tokens_mut(),
+            Annotatable::ForeignItem(foreign_item) => foreign_item.tokens_mut(),
+            Annotatable::Stmt(stmt) => stmt.tokens_mut(),
+            Annotatable::Expr(expr) => expr.tokens_mut(),
+            Annotatable::Arm(arm) => arm.tokens_mut(),
+            Annotatable::Field(field) => field.tokens_mut(),
+            Annotatable::FieldPat(fp) => fp.tokens_mut(),
+            Annotatable::GenericParam(gp) => gp.tokens_mut(),
+            Annotatable::Param(p) => p.tokens_mut(),
+            Annotatable::StructField(sf) => sf.tokens_mut(),
+            Annotatable::Variant(v) => v.tokens_mut(),
+        }
     }
 }
 
diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs
@@ -72,6 +72,10 @@ impl<'a> Parser<'a> {
         let cursor_snapshot = self.token_cursor.clone();
 
         let (mut ret, trailing_token) = f(self, attrs.attrs)?;
+        let tokens = match ret.tokens_mut() {
+            Some(tokens) if tokens.is_none() => tokens,
+            _ => return Ok(ret),
+        };
 
         // Produces a `TokenStream` on-demand. Using `cursor_snapshot`
         // and `num_calls`, we can reconstruct the `TokenStream` seen
@@ -128,14 +132,14 @@ impl<'a> Parser<'a> {
             }
         }
 
-        let lazy_impl = LazyTokenStreamImpl {
+        *tokens = Some(LazyTokenStream::new(LazyTokenStreamImpl {
             start_token,
             num_calls,
             cursor_snapshot,
             desugar_doc_comments: self.desugar_doc_comments,
             append_unglued_token: self.token_cursor.append_unglued_token.clone(),
-        };
-        ret.finalize_tokens(LazyTokenStream::new(lazy_impl));
+        }));
+
         Ok(ret)
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -11,13 +11,7 @@ use super::{AttrVec, Attribute, Stmt, StmtKind};`
`11`	`11`	`pub trait AstLike: Sized {`
`12`	`12`	`fn attrs(&self) -> &[Attribute];`
`13`	`13`	`fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>));`
`14`		- /// Called by `Parser::collect_tokens` to store the collected
`15`		`- /// tokens inside an AST node`
`16`		`- fn finalize_tokens(&mut self, _tokens: LazyTokenStream) {`
`17`		`- // This default impl makes this trait easier to implement`
`18`		- // in tools like `rust-analyzer`
`19`		- panic!("`finalize_tokens` is not supported!")
`20`		`- }`
	`14`	`+ fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>>;`
`21`	`15`	`}`
`22`	`16`
`23`	`17`	`impl<T: AstLike + 'static> AstLike for P<T> {`
`@@ -27,8 +21,8 @@ impl<T: AstLike + 'static> AstLike for P<T> {`
`27`	`21`	`fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {`
`28`	`22`	`(**self).visit_attrs(f);`
`29`	`23`	`}`
`30`		`- fn finalize_tokens(&mut self, tokens: LazyTokenStream) {`
`31`		`- (**self).finalize_tokens(tokens)`
	`24`	`+ fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {`
	`25`	`+ (**self).tokens_mut()`
`32`	`26`	`}`
`33`	`27`	`}`
`34`	`28`
`@@ -42,12 +36,12 @@ fn visit_attrvec(attrs: &mut AttrVec, f: impl FnOnce(&mut Vec<Attribute>)) {`
`42`	`36`
`43`	`37`	`impl AstLike for StmtKind {`
`44`	`38`	`fn attrs(&self) -> &[Attribute] {`
`45`		`- match *self {`
`46`		`- StmtKind::Local(ref local) => local.attrs(),`
`47`		`- StmtKind::Expr(ref expr) \| StmtKind::Semi(ref expr) => expr.attrs(),`
`48`		`- StmtKind::Item(ref item) => item.attrs(),`
	`39`	`+ match self {`
	`40`	`+ StmtKind::Local(local) => local.attrs(),`
	`41`	`+ StmtKind::Expr(expr) \| StmtKind::Semi(expr) => expr.attrs(),`
	`42`	`+ StmtKind::Item(item) => item.attrs(),`
`49`	`43`	`StmtKind::Empty => &[],`
`50`		`- StmtKind::MacCall(ref mac) => &*mac.attrs,`
	`44`	`+ StmtKind::MacCall(mac) => &mac.attrs,`
`51`	`45`	`}`
`52`	`46`	`}`
`53`	`47`
`@@ -60,17 +54,14 @@ impl AstLike for StmtKind {`
`60`	`54`	`StmtKind::MacCall(mac) => visit_attrvec(&mut mac.attrs, f),`
`61`	`55`	`}`
`62`	`56`	`}`
`63`		`- fn finalize_tokens(&mut self, tokens: LazyTokenStream) {`
`64`		`- let stmt_tokens = match self {`
`65`		`- StmtKind::Local(ref mut local) => &mut local.tokens,`
`66`		`- StmtKind::Item(ref mut item) => &mut item.tokens,`
`67`		`- StmtKind::Expr(ref mut expr) \| StmtKind::Semi(ref mut expr) => &mut expr.tokens,`
`68`		`- StmtKind::Empty => return,`
`69`		`- StmtKind::MacCall(ref mut mac) => &mut mac.tokens,`
`70`		`- };`
`71`		`- if stmt_tokens.is_none() {`
`72`		`- *stmt_tokens = Some(tokens);`
`73`		`- }`
	`57`	`+ fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {`
	`58`	`+ Some(match self {`
	`59`	`+ StmtKind::Local(local) => &mut local.tokens,`
	`60`	`+ StmtKind::Item(item) => &mut item.tokens,`
	`61`	`+ StmtKind::Expr(expr) \| StmtKind::Semi(expr) => &mut expr.tokens,`
	`62`	`+ StmtKind::Empty => return None,`
	`63`	`+ StmtKind::MacCall(mac) => &mut mac.tokens,`
	`64`	`+ })`
`74`	`65`	`}`
`75`	`66`	`}`
`76`	`67`
`@@ -82,8 +73,8 @@ impl AstLike for Stmt {`
`82`	`73`	`fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {`
`83`	`74`	`self.kind.visit_attrs(f);`
`84`	`75`	`}`
`85`		`- fn finalize_tokens(&mut self, tokens: LazyTokenStream) {`
`86`		`- self.kind.finalize_tokens(tokens)`
	`76`	`+ fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {`
	`77`	`+ self.kind.tokens_mut()`
`87`	`78`	`}`
`88`	`79`	`}`
`89`	`80`
`@@ -92,17 +83,13 @@ impl AstLike for Attribute {`
`92`	`83`	`&[]`
`93`	`84`	`}`
`94`	`85`	`fn visit_attrs(&mut self, _f: impl FnOnce(&mut Vec<Attribute>)) {}`
`95`		`- fn finalize_tokens(&mut self, tokens: LazyTokenStream) {`
`96`		`- match &mut self.kind {`
`97`		`- AttrKind::Normal(_, attr_tokens) => {`
`98`		`- if attr_tokens.is_none() {`
`99`		`- *attr_tokens = Some(tokens);`
`100`		`- }`
	`86`	`+ fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {`
	`87`	`+ Some(match &mut self.kind {`
	`88`	`+ AttrKind::Normal(_, tokens) => tokens,`
	`89`	`+ kind @ AttrKind::DocComment(..) => {`
	`90`	`+ panic!("Called tokens_mut on doc comment attr {:?}", kind)`
`101`	`91`	`}`
`102`		`- AttrKind::DocComment(..) => {`
`103`		`- panic!("Called finalize_tokens on doc comment attr {:?}", self)`
`104`		`- }`
`105`		`- }`
	`92`	`+ })`
`106`	`93`	`}`
`107`	`94`	`}`
`108`	`95`
`@@ -115,10 +102,8 @@ impl<T: AstLike> AstLike for Option<T> {`
`115`	`102`	`inner.visit_attrs(f);`
`116`	`103`	`}`
`117`	`104`	`}`
`118`		`- fn finalize_tokens(&mut self, tokens: LazyTokenStream) {`
`119`		`- if let Some(inner) = self {`
`120`		`- inner.finalize_tokens(tokens);`
`121`		`- }`
	`105`	`+ fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {`
	`106`	`+ self.as_mut().and_then(\|inner\| inner.tokens_mut())`
`122`	`107`	`}`
`123`	`108`	`}`
`124`	`109`
`@@ -152,11 +137,8 @@ macro_rules! derive_has_tokens_and_attrs {`
`152`	`137`	`VecOrAttrVec::visit(&mut self.attrs, f)`
`153`	`138`	`}`
`154`	`139`
`155`		`- fn finalize_tokens(&mut self, tokens: LazyTokenStream) {`
`156`		`- if self.tokens.is_none() {`
`157`		`- self.tokens = Some(tokens);`
`158`		`- }`
`159`		`-`
	`140`	`+ fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {`
	`141`	`+ Some(&mut self.tokens)`
`160`	`142`	`}`
`161`	`143`	`}`
`162`	`144`	`)* }`
`@@ -173,7 +155,9 @@ macro_rules! derive_has_attrs_no_tokens {`
`173`	`155`	`VecOrAttrVec::visit(&mut self.attrs, f)`
`174`	`156`	`}`
`175`	`157`
`176`		`- fn finalize_tokens(&mut self, _tokens: LazyTokenStream) {}`
	`158`	`+ fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {`
	`159`	`+ None`
	`160`	`+ }`
`177`	`161`	`}`
`178`	`162`	`)* }`
`179`	`163`	`}`
`@@ -185,14 +169,10 @@ macro_rules! derive_has_tokens_no_attrs {`
`185`	`169`	`&[]`
`186`	`170`	`}`
`187`	`171`
`188`		`- fn visit_attrs(&mut self, _f: impl FnOnce(&mut Vec<Attribute>)) {`
`189`		`- }`
`190`		`-`
`191`		`- fn finalize_tokens(&mut self, tokens: LazyTokenStream) {`
`192`		`- if self.tokens.is_none() {`
`193`		`- self.tokens = Some(tokens);`
`194`		`- }`
	`172`	`+ fn visit_attrs(&mut self, _f: impl FnOnce(&mut Vec<Attribute>)) {}`
`195`	`173`
	`174`	`+ fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {`
	`175`	`+ Some(&mut self.tokens)`
`196`	`176`	`}`
`197`	`177`	`}`
`198`	`178`	`)* }`
Original file line number	Diff line number	Diff line change
`@@ -81,8 +81,22 @@ impl AstLike for Annotatable {`
`81`	`81`	`}`
`82`	`82`	`}`
`83`	`83`
`84`		`- fn finalize_tokens(&mut self, tokens: LazyTokenStream) {`
`85`		`- panic!("Called finalize_tokens on an Annotatable: {:?}", tokens);`
	`84`	`+ fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {`
	`85`	`+ match self {`
	`86`	`+ Annotatable::Item(item) => item.tokens_mut(),`
	`87`	`+ Annotatable::TraitItem(trait_item) => trait_item.tokens_mut(),`
	`88`	`+ Annotatable::ImplItem(impl_item) => impl_item.tokens_mut(),`
	`89`	`+ Annotatable::ForeignItem(foreign_item) => foreign_item.tokens_mut(),`
	`90`	`+ Annotatable::Stmt(stmt) => stmt.tokens_mut(),`
	`91`	`+ Annotatable::Expr(expr) => expr.tokens_mut(),`
	`92`	`+ Annotatable::Arm(arm) => arm.tokens_mut(),`
	`93`	`+ Annotatable::Field(field) => field.tokens_mut(),`
	`94`	`+ Annotatable::FieldPat(fp) => fp.tokens_mut(),`
	`95`	`+ Annotatable::GenericParam(gp) => gp.tokens_mut(),`
	`96`	`+ Annotatable::Param(p) => p.tokens_mut(),`
	`97`	`+ Annotatable::StructField(sf) => sf.tokens_mut(),`
	`98`	`+ Annotatable::Variant(v) => v.tokens_mut(),`
	`99`	`+ }`
`86`	`100`	`}`
`87`	`101`	`}`
`88`	`102`