Treat k#ident keywords as valid tokens

Author: fee1-dead

compiler/rustc_ast/src/token.rs

@@ -299,6 +299,10 @@ pub enum TokenKind {
     /// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to
     /// treat regular and interpolated identifiers in the same way.
     Ident(Symbol, /* is_raw */ bool),
+
+    /// A `k#ident` keyword
+    Keyword(Symbol),
+
     /// Lifetime identifier token.
     /// Do not forget about `NtLifetime` when you want to match on lifetime identifiers.
     /// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to
@@ -438,7 +442,7 @@ impl Token {
             | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar | Question | SingleQuote => true,
 
             OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
-            | Lifetime(..) | Interpolated(..) | Eof => false,
+            | Keyword(..) | Lifetime(..) | Interpolated(..) | Eof => false,
         }
     }
 
@@ -821,7 +825,7 @@ impl Token {
 
             Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot
             | DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar
-            | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..)
+            | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | Keyword(..)
             | Lifetime(..) | Interpolated(..) | DocComment(..) | Eof => return None,
         };
 

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -781,6 +781,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
             token::Ident(s, is_raw) => {
                 IdentPrinter::new(s, is_raw, convert_dollar_crate).to_string().into()
             }
+            token::Keyword(s) => format!("k#{s}").into(),
             token::Lifetime(s) => s.to_string().into(),
 
             /* Other */

compiler/rustc_expand/src/proc_macro_server.rs

@@ -203,6 +203,11 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
                 SingleQuote => op("'"),
 
                 Ident(sym, is_raw) => trees.push(TokenTree::Ident(Ident { sym, is_raw, span })),
+                Keyword(sym) => trees.extend([
+                    TokenTree::Ident(Ident { sym: sym::k, is_raw: false, span }),
+                    TokenTree::Punct(Punct { ch: b'#', joint: true, span }),
+                    TokenTree::Ident(Ident { sym, is_raw: false, span }),
+                ]),
                 Lifetime(name) => {
                     let ident = symbol::Ident::new(name, span).without_first_quote();
                     trees.extend([

compiler/rustc_lexer/src/lib.rs

@@ -78,6 +78,9 @@ pub enum TokenKind {
     /// "r#ident"
     RawIdent,
 
+    /// "k#ident"
+    KeywordIdent,
+
     /// An unknown prefix, like `foo#`, `foo'`, `foo"`.
     ///
     /// Note that only the
@@ -375,6 +378,11 @@ impl Cursor<'_> {
                 None,
             ),
 
+            'k' => match (self.first(), self.second()) {
+                ('#', c1) if is_id_start(c1) => self.keyword_ident(),
+                _ => self.ident_or_unknown_prefix(),
+            },
+
             // Identifier (this should be checked after other variant that can
             // start as identifier).
             c if is_id_start(c) => self.ident_or_unknown_prefix(),
@@ -505,6 +513,15 @@ impl Cursor<'_> {
         RawIdent
     }
 
+    fn keyword_ident(&mut self) -> TokenKind {
+        debug_assert!(self.prev() == 'k' && self.first() == '#' && is_id_start(self.second()));
+        // Eat "#" symbol.
+        self.bump();
+        // Eat the identifier part of KeywordIdent.
+        self.eat_identifier();
+        KeywordIdent
+    }
+
     fn ident_or_unknown_prefix(&mut self) -> TokenKind {
         debug_assert!(is_id_start(self.prev()));
         // Start is already eaten, eat the rest of identifier.

compiler/rustc_parse/src/lexer/mod.rs

@@ -185,6 +185,18 @@ impl<'a> StringReader<'a> {
                     self.sess.raw_identifier_spans.push(span);
                     token::Ident(sym, true)
                 }
+                // Treat `k#ident` as a normal identifier token before 2021.
+                rustc_lexer::TokenKind::KeywordIdent if !self.mk_sp(start, self.pos).edition().at_least_rust_2021() => {
+                    // FIXME: what should we do when this is a _known_ prefix?
+                    self.report_unknown_prefix(start);
+                    self.ident(start)
+                }
+                rustc_lexer::TokenKind::KeywordIdent => {
+                    let sym = nfc_normalize(self.str_from(start));
+                    let span = self.mk_sp(start, self.pos);
+                    self.sess.symbol_gallery.insert(sym, span);
+                    token::Keyword(sym)
+                }
                 rustc_lexer::TokenKind::UnknownPrefix => {
                     self.report_unknown_prefix(start);
                     self.ident(start)

compiler/rustc_span/src/symbol.rs

@@ -917,6 +917,7 @@ symbols! {
         iter_repeat,
         iterator,
         iterator_collect_fn,
+        k,
         kcfi,
         keyword,
         kind,

src/librustdoc/html/highlight.rs

@@ -878,6 +878,7 @@ impl<'src> Classifier<'src> {
             TokenKind::RawIdent | TokenKind::UnknownPrefix | TokenKind::InvalidIdent => {
                 Class::Ident(self.new_span(before, text))
             }
+            TokenKind::KeywordIdent => Class::KeyWord,
             TokenKind::Lifetime { .. } => Class::Lifetime,
             TokenKind::Eof => panic!("Eof in advance"),
         };

src/tools/rust-analyzer/crates/parser/src/lexed_str.rs

@@ -224,6 +224,7 @@ impl<'a> Converter<'a> {
                 rustc_lexer::TokenKind::Slash => T![/],
                 rustc_lexer::TokenKind::Caret => T![^],
                 rustc_lexer::TokenKind::Percent => T![%],
+                rustc_lexer::TokenKind::KeywordIdent => ERROR,
                 rustc_lexer::TokenKind::Unknown => ERROR,
                 rustc_lexer::TokenKind::UnknownPrefix if token_text == "builtin" => IDENT,
                 rustc_lexer::TokenKind::UnknownPrefix => {