internal: Derive kinds information from ungrammar file

crates/hir-def/src/body/lower.rs

@@ -301,7 +301,10 @@ impl ExprCollector<'_> {
                         result_expr_id
                     })
                 }
-                None => self.collect_block(e),
+                // FIXME
+                Some(ast::BlockModifier::AsyncGen(_)) | Some(ast::BlockModifier::Gen(_)) | None => {
+                    self.collect_block(e)
+                }
             },
             ast::Expr::LoopExpr(e) => {
                 let label = e.label().map(|label| self.collect_label(label));

crates/hir-expand/src/builtin_fn_macro.rs

@@ -728,6 +728,8 @@ fn include_expand(
         }
     };
     match parse_to_token_tree(
+        // FIXME
+        Edition::CURRENT,
         SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID },
         SyntaxContextId::ROOT,
         &db.file_text(file_id),

crates/ide-assists/src/handlers/extract_variable.rs

@@ -3,7 +3,7 @@ use syntax::{
     ast::{self, edit::IndentLevel, edit_in_place::Indent, make, AstNode, HasName},
     ted, NodeOrToken,
     SyntaxKind::{BLOCK_EXPR, BREAK_EXPR, COMMENT, LOOP_EXPR, MATCH_GUARD, PATH_EXPR, RETURN_EXPR},
-    SyntaxNode,
+    SyntaxNode, T,
 };
 
 use crate::{utils::suggest_name, AssistContext, AssistId, AssistKind, Assists};
@@ -26,8 +26,8 @@ use crate::{utils::suggest_name, AssistContext, AssistId, AssistKind, Assists};
 // ```
 pub(crate) fn extract_variable(acc: &mut Assists, ctx: &AssistContext<'_>) -> Option<()> {
     let node = if ctx.has_empty_selection() {
-        if let Some(expr_stmt) = ctx.find_node_at_offset::<ast::ExprStmt>() {
-            expr_stmt.syntax().clone()
+        if let Some(t) = ctx.token_at_offset().find(|it| it.kind() == T![;]) {
+            t.parent().and_then(ast::ExprStmt::cast)?.syntax().clone()
         } else if let Some(expr) = ctx.find_node_at_offset::<ast::Expr>() {
             expr.syntax().ancestors().find_map(valid_target_expr)?.syntax().clone()
         } else {

crates/ide-assists/src/utils/suggest_name.rs

@@ -184,7 +184,7 @@ fn normalize(name: &str) -> Option<String> {
 
 fn is_valid_name(name: &str) -> bool {
     matches!(
-        ide_db::syntax_helpers::LexedStr::single_token(name),
+        ide_db::syntax_helpers::LexedStr::single_token(syntax::Edition::CURRENT, name),
         Some((syntax::SyntaxKind::IDENT, _error))
     )
 }

crates/ide-db/src/rename.rs

@@ -25,7 +25,7 @@ use std::fmt;
 use base_db::{AnchoredPathBuf, FileId, FileRange};
 use either::Either;
 use hir::{FieldSource, HirFileIdExt, InFile, ModuleSource, Semantics};
-use span::SyntaxContextId;
+use span::{Edition, SyntaxContextId};
 use stdx::{never, TupleExt};
 use syntax::{
     ast::{self, HasName},
@@ -227,7 +227,8 @@ fn rename_mod(
     module: hir::Module,
     new_name: &str,
 ) -> Result<SourceChange> {
-    if IdentifierKind::classify(new_name)? != IdentifierKind::Ident {
+    if IdentifierKind::classify(module.krate().edition(sema.db), new_name)? != IdentifierKind::Ident
+    {
         bail!("Invalid name `{0}`: cannot rename module to {0}", new_name);
     }
 
@@ -313,7 +314,12 @@ fn rename_reference(
     def: Definition,
     new_name: &str,
 ) -> Result<SourceChange> {
-    let ident_kind = IdentifierKind::classify(new_name)?;
+    let ident_kind = IdentifierKind::classify(
+        def.krate(sema.db)
+            .ok_or_else(|| RenameError("definition has no krate?".into()))?
+            .edition(sema.db),
+        new_name,
+    )?;
 
     if matches!(
         def,
@@ -605,8 +611,8 @@ pub enum IdentifierKind {
 }
 
 impl IdentifierKind {
-    pub fn classify(new_name: &str) -> Result<IdentifierKind> {
-        match parser::LexedStr::single_token(new_name) {
+    pub fn classify(edition: Edition, new_name: &str) -> Result<IdentifierKind> {
+        match parser::LexedStr::single_token(edition, new_name) {
             Some(res) => match res {
                 (SyntaxKind::IDENT, _) => {
                     if let Some(inner) = new_name.strip_prefix("r#") {

crates/ide-db/src/syntax_helpers/node_ext.rs

@@ -277,6 +277,8 @@ pub fn for_each_tail_expr(expr: &ast::Expr, cb: &mut dyn FnMut(&ast::Expr)) {
                     });
                 }
                 Some(ast::BlockModifier::Unsafe(_)) => (),
+                Some(ast::BlockModifier::Gen(_)) => (),
+                Some(ast::BlockModifier::AsyncGen(_)) => (),
                 None => (),
             }
             if let Some(stmt_list) = b.stmt_list() {

crates/ide-ssr/src/parsing.rs

@@ -255,7 +255,7 @@ fn validate_rule(rule: &SsrRule) -> Result<(), SsrError> {
 }
 
 fn tokenize(source: &str) -> Result<Vec<Token>, SsrError> {
-    let lexed = parser::LexedStr::new(source);
+    let lexed = parser::LexedStr::new(parser::Edition::CURRENT, source);
     if let Some((_, first_error)) = lexed.errors().next() {
         bail!("Failed to parse pattern: {}", first_error);
     }

crates/ide/src/rename.rs

@@ -13,6 +13,7 @@ use ide_db::{
     RootDatabase,
 };
 use itertools::Itertools;
+use span::Edition;
 use stdx::{always, never};
 use syntax::{
     ast, utils::is_raw_identifier, AstNode, SmolStr, SyntaxKind, SyntaxNode, TextRange, TextSize,
@@ -99,7 +100,7 @@ pub(crate) fn rename(
             // FIXME: This can use the `ide_db::rename_reference` (or def.rename) method once we can
             // properly find "direct" usages/references.
             .map(|(.., def)| {
-                match IdentifierKind::classify(new_name)? {
+                match IdentifierKind::classify(Edition::CURRENT, new_name)? {
                     IdentifierKind::Ident => (),
                     IdentifierKind::Lifetime => {
                         bail!("Cannot alias reference to a lifetime identifier")
@@ -391,7 +392,7 @@ fn rename_self_to_param(
         return Ok(SourceChange::default());
     }
 
-    let identifier_kind = IdentifierKind::classify(new_name)?;
+    let identifier_kind = IdentifierKind::classify(Edition::CURRENT, new_name)?;
 
     let InFile { file_id, value: self_param } =
         sema.source(self_param).ok_or_else(|| format_err!("cannot find function source"))?;

crates/mbe/src/syntax_bridge.rs

@@ -169,6 +169,7 @@ where
 /// Convert a string to a `TokenTree`. The spans of the subtree will be anchored to the provided
 /// anchor with the given context.
 pub fn parse_to_token_tree<Ctx>(
+    edition: Edition,
     anchor: SpanAnchor,
     ctx: Ctx,
     text: &str,
@@ -177,7 +178,7 @@ where
     SpanData<Ctx>: Copy + fmt::Debug,
     Ctx: Copy,
 {
-    let lexed = parser::LexedStr::new(text);
+    let lexed = parser::LexedStr::new(edition, text);
     if lexed.errors().next().is_some() {
         return None;
     }
@@ -187,11 +188,15 @@ where
 }
 
 /// Convert a string to a `TokenTree`. The passed span will be used for all spans of the produced subtree.
-pub fn parse_to_token_tree_static_span<S>(span: S, text: &str) -> Option<tt::Subtree<S>>
+pub fn parse_to_token_tree_static_span<S>(
+    edition: Edition,
+    span: S,
+    text: &str,
+) -> Option<tt::Subtree<S>>
 where
     S: Copy + fmt::Debug,
 {
-    let lexed = parser::LexedStr::new(text);
+    let lexed = parser::LexedStr::new(edition, text);
     if lexed.errors().next().is_some() {
         return None;
     }

crates/parser/src/edition.rs

@@ -12,8 +12,17 @@ pub enum Edition {
 }
 
 impl Edition {
+    /// The current latest stable edition, note this is usually not the right choice in code.
     pub const CURRENT: Edition = Edition::Edition2021;
     pub const DEFAULT: Edition = Edition::Edition2015;
+
+    pub fn at_least_2024(self) -> bool {
+        self >= Edition::Edition2024
+    }
+
+    pub fn at_least_2018(self) -> bool {
+        self >= Edition::Edition2018
+    }
 }
 
 #[derive(Debug)]

crates/parser/src/grammar.rs

@@ -165,42 +165,6 @@ pub(crate) mod entry {
             }
             m.complete(p, ERROR);
         }
-
-        pub(crate) fn eager_macro_input(p: &mut Parser<'_>) {
-            let m = p.start();
-
-            let closing_paren_kind = match p.current() {
-                T!['{'] => T!['}'],
-                T!['('] => T![')'],
-                T!['['] => T![']'],
-                _ => {
-                    p.error("expected `{`, `[`, `(`");
-                    while !p.at(EOF) {
-                        p.bump_any();
-                    }
-                    m.complete(p, ERROR);
-                    return;
-                }
-            };
-            p.bump_any();
-            while !p.at(EOF) && !p.at(closing_paren_kind) {
-                if expressions::expr(p).is_none() {
-                    break;
-                }
-                if !p.at(EOF) && !p.at(closing_paren_kind) {
-                    p.expect(T![,]);
-                }
-            }
-            p.expect(closing_paren_kind);
-            if p.at(EOF) {
-                m.complete(p, MACRO_EAGER_INPUT);
-                return;
-            }
-            while !p.at(EOF) {
-                p.bump_any();
-            }
-            m.complete(p, ERROR);
-        }
     }
 }
 

crates/parser/src/grammar/expressions/atom.rs

@@ -51,6 +51,7 @@ pub(super) const ATOM_EXPR_FIRST: TokenSet =
         T![const],
         T![continue],
         T![do],
+        T![gen],
         T![for],
         T![if],
         T![let],
@@ -138,15 +139,37 @@ pub(super) fn atom_expr(
         // fn f() { const { } }
         // fn f() { async { } }
         // fn f() { async move { } }
-        T![const] | T![unsafe] | T![async] if la == T!['{'] => {
+        T![const] | T![unsafe] | T![async] | T![gen] if la == T!['{'] => {
             let m = p.start();
             p.bump_any();
             stmt_list(p);
             m.complete(p, BLOCK_EXPR)
         }
-        T![async] if la == T![move] && p.nth(2) == T!['{'] => {
+        // test_err gen_blocks
+        // pub fn main() {
+        //     gen { yield ""; };
+        //     async gen { yield ""; };
+        //     gen move { yield ""; };
+        //     async gen move { yield ""; };
+        // }
+        T![async] if la == T![gen] && p.nth(2) == T!['{'] => {
+            let m = p.start();
+            p.bump(T![async]);
+            p.eat(T![gen]);
+            stmt_list(p);
+            m.complete(p, BLOCK_EXPR)
+        }
+        T![async] | T![gen] if la == T![move] && p.nth(2) == T!['{'] => {
+            let m = p.start();
+            p.bump_any();
+            p.bump(T![move]);
+            stmt_list(p);
+            m.complete(p, BLOCK_EXPR)
+        }
+        T![async] if la == T![gen] && p.nth(2) == T![move] && p.nth(3) == T!['{'] => {
             let m = p.start();
             p.bump(T![async]);
+            p.bump(T![gen]);
             p.bump(T![move]);
             stmt_list(p);
             m.complete(p, BLOCK_EXPR)
@@ -355,6 +378,7 @@ fn closure_expr(p: &mut Parser<'_>) -> CompletedMarker {
     p.eat(T![const]);
     p.eat(T![static]);
     p.eat(T![async]);
+    p.eat(T![gen]);
     p.eat(T![move]);
 
     if !p.at(T![|]) {

crates/parser/src/grammar/items.rs

@@ -112,11 +112,22 @@ pub(super) fn opt_item(p: &mut Parser<'_>, m: Marker) -> Result<(), Marker> {
 
     // test_err async_without_semicolon
     // fn foo() { let _ = async {} }
-    if p.at(T![async]) && !matches!(p.nth(1), T!['{'] | T![move] | T![|]) {
+    if p.at(T![async])
+        && (!matches!(p.nth(1), T!['{'] | T![gen] | T![move] | T![|])
+            || matches!((p.nth(1), p.nth(2)), (T![gen], T![fn])))
+    {
         p.eat(T![async]);
         has_mods = true;
     }
 
+    // test_err gen_fn
+    // gen fn gen_fn() {}
+    // async gen fn async_gen_fn() {}
+    if p.at(T![gen]) && p.nth(1) == T![fn] {
+        p.eat(T![gen]);
+        has_mods = true;
+    }
+
     // test_err unsafe_block_in_mod
     // fn foo(){} unsafe { } fn bar(){}
     if p.at(T![unsafe]) && p.nth(1) != T!['{'] {
@@ -173,13 +184,6 @@ pub(super) fn opt_item(p: &mut Parser<'_>, m: Marker) -> Result<(), Marker> {
         }
     }
 
-    // test existential_type
-    // existential type Foo: Fn() -> usize;
-    if p.at_contextual_kw(T![existential]) && p.nth(1) == T![type] {
-        p.bump_remap(T![existential]);
-        has_mods = true;
-    }
-
     // items
     match p.current() {
         T![fn] => fn_(p, m),
@@ -201,7 +205,7 @@ pub(super) fn opt_item(p: &mut Parser<'_>, m: Marker) -> Result<(), Marker> {
 
         _ if has_visibility || has_mods => {
             if has_mods {
-                p.error("expected existential, fn, trait or impl");
+                p.error("expected fn, trait or impl");
             } else {
                 p.error("expected an item");
             }