fix(devin-lang): add support for variable and command identifiers in grammar and lexer #101

The commit introduces new token types for variable ($VAR) and command (COMMAND) identifiers, as well as an agent identifier (AGENT). This change also includes a new state in the lexer to handle content parsing more efficiently.

Author: phodal

docs/scenes/legacy-migration.md

@@ -20,6 +20,8 @@ Building upon the experience accumulated in our "[System Refactoring and Migrati
 
 Support language: Java.
 
+Screenshot EXAMPLE：
+
 ![AutoDev Living Documentation](https://unitmesh.cc/auto-dev/gen-test-data.png)
 
 Simply right-click on the Java Controller code, select "Generate Test Data (APIs)," and API test data will be generated.
@@ -51,6 +53,8 @@ Select the corresponding code, right-click, and choose `Generate Documentation`
 
 ## Use [Custom Living Documentation](/custom/living-documentation) for generating dynamic documentation.
 
+Screenshot EXAMPLE：
+
 ![AutoDev Living Documentation](https://unitmesh.cc/auto-dev/autodev-living-doc.png)
 
 Configuration: `Settings` -> `Tools` -> `AutoDev` -> `Customize Engine prompt`, add custom formats for living documentation. Example:
@@ -87,6 +91,6 @@ AutoDev has been supporting basic PL/SQL migration since version 1.5.5.
    - Generate Test Cases
    - Generate Java Code
 
-Screenshot:
+Screenshot EXAMPLE：
 
 ![SQL Migration](https://unitmesh.cc/auto-dev/autodev-sql-migration.png)

exts/devin-lang/src/grammar/DevInLexer.flex

@@ -22,9 +22,11 @@ import com.intellij.psi.TokenType;
 %eof{  return;
 %eof}
 
+%s YYUSED
 %s AGENT_BLOCK
 %s VARIABLE_BLOCK
 %s COMMAND_BLOCK
+
 %s CODE_BLOCK
 %s LANG_ID
 
@@ -65,21 +67,51 @@ NEWLINE= \n | \r | \r\n
 
         return CODE_CONTENT;
     }
+
+    private IElementType content() {
+        String text = yytext().toString().trim();
+        if (isCodeStart == true && text.equals("```")) {
+            return codeContent();
+        }
+
+        if (isCodeStart == false && text.startsWith("```")) {
+            isCodeStart = true;
+            yypushback(yylength() - 3);
+            yybegin(LANG_ID);
+
+            return CODE_BLOCK_START;
+        }
+
+        if (isCodeStart) {
+            return CODE_CONTENT;
+        } else {
+            yypushback(yylength());
+            yybegin(YYUSED);
+
+            return TEXT_SEGMENT;
+        }
+    }
 %}
 
 %%
 <YYINITIAL> {
-  "@"                  { if(!isCodeStart) { yybegin(AGENT_BLOCK); return AGENT_START; } else { yypushback(1); yybegin(CODE_BLOCK); }}
-  "/"                  { if(!isCodeStart) { yybegin(COMMAND_BLOCK); return COMMAND_START; } else { yypushback(1); yybegin(CODE_BLOCK); }}
-  "$"                  { if(!isCodeStart) { yybegin(VARIABLE_BLOCK); return VARIABLE_START; } else { yypushback(1); yybegin(CODE_BLOCK); }}
-
-  "```" {IDENTIFIER}?  { yybegin(LANG_ID); if (isCodeStart == true) { isCodeStart = false; return CODE_BLOCK_END; } else { isCodeStart = true; }; yypushback(yylength()); }
-
-  {TEXT_SEGMENT}       { if(isCodeStart) { return codeContent(); } else { return TEXT_SEGMENT; } }
+  {CODE_CONTENT}       { return content(); }
   {NEWLINE}            { return NEWLINE;  }
   [^]                  { return TokenType.BAD_CHARACTER; }
 }
 
+<YYUSED> {
+  "@"                     { yybegin(AGENT_BLOCK); return AGENT_START; }
+  "/"                     { yybegin(COMMAND_BLOCK); return COMMAND_START; }
+  "$"                     { yybegin(VARIABLE_BLOCK); return VARIABLE_START; }
+
+  "```" {IDENTIFIER}?     { yybegin(LANG_ID); if (isCodeStart == true) { isCodeStart = false; return CODE_BLOCK_END; } else { isCodeStart = true; }; yypushback(yylength()); }
+
+  {NEWLINE}               { return NEWLINE; }
+  {TEXT_SEGMENT}          { return TEXT_SEGMENT; }
+  [^]                     {  }
+}
+
 <AGENT_BLOCK> {
   {IDENTIFIER}         { yybegin(YYINITIAL); return AGENT_ID; }
   [^]                  { return TokenType.BAD_CHARACTER; }

exts/devin-lang/src/grammar/DevInParser.bnf

@@ -24,6 +24,9 @@
     CODE_CONTENT       = "CODE_CONTENT"
     IDENTIFIER         = "IDENTIFIER"
     LANGUAGE_ID        = "LANGUAGE_ID"
+    VARIABLE_ID        = "VARIABLE_ID"
+    COMMAND_ID         = "COMMAND_ID"
+    AGENT_ID           = "AGENT_ID"
   ]
 }
 

exts/devin-lang/src/test/testData/parser/JavaAnnotation.txt

@@ -8,8 +8,7 @@ DevInFile
       PsiElement(DevInTokenType.NEWLINE)('\n')
       PsiElement(DevInTokenType.CODE_CONTENT)('@Retention(RetentionPolicy.RUNTIME)')
       PsiElement(DevInTokenType.NEWLINE)('\n')
-      PsiElement(DevInTokenType.CODE_CONTENT)('public ')
-      PsiElement(DevInTokenType.CODE_CONTENT)('@interface ExampleAnnotation {')
+      PsiElement(DevInTokenType.CODE_CONTENT)('public @interface ExampleAnnotation {')
       PsiElement(DevInTokenType.NEWLINE)('\n')
       PsiElement(DevInTokenType.CODE_CONTENT)('    String value() default "";')
       PsiElement(DevInTokenType.NEWLINE)('\n')