feat(devin-lang): improve code fence parsing to support embedded languages #101

The commit introduces a new feature to the DevInLexer.flex file, enhancing the code fence parsing functionality to recognize and handle embedded languages within code blocks. This change enables the lexer to correctly interpret content inside code fences, which is crucial for the correct parsing and processing of code samples in the DevInLang language.

The modification includes the addition of a new state, `CODE_BLOCK`, to distinguish between normal text segments and code blocks. The lexer now checks for the start of a code block with a leading `@`, `/`, or `$` character, and only transitions to the corresponding block type (e.g., `AGENT_BLOCK`, `COMMAND_BLOCK`, or `VARIABLE_BLOCK`) if the current input is not within a code block.

Furthermore, a new test case is added to `DevInParsingTest.kt` to validate the parsing of Java annotations within code blocks, ensuring the robustness of the lexer's new behavior.

Author: phodal

exts/devin-lang/src/grammar/DevInLexer.flex

@@ -36,7 +36,7 @@ COMMAND_ID=[a-zA-Z0-9][_\-a-zA-Z0-9]*
 LANGUAGE_ID=[a-zA-Z0-9][_\-a-zA-Z0-9]*
 
 TEXT_SEGMENT=[^$/@\n]+
-CODE_CONTENT=([^$/@\n]+ )
+CODE_CONTENT=[^\n]+
 NEWLINE= \n | \r | \r\n
 
 %{
@@ -69,9 +69,10 @@ NEWLINE= \n | \r | \r\n
 
 %%
 <YYINITIAL> {
-  "@"                  { yybegin(AGENT_BLOCK);    return AGENT_START; }
-  "/"                  { yybegin(COMMAND_BLOCK);  return COMMAND_START; }
-  "$"                  { yybegin(VARIABLE_BLOCK); return VARIABLE_START; }
+  "@"                  { if(!isCodeStart) { yybegin(AGENT_BLOCK); return AGENT_START; } else { yypushback(1); yybegin(CODE_BLOCK); }}
+  "/"                  { if(!isCodeStart) { yybegin(COMMAND_BLOCK); return COMMAND_START; } else { yypushback(1); yybegin(CODE_BLOCK); }}
+  "$"                  { if(!isCodeStart) { yybegin(VARIABLE_BLOCK); return VARIABLE_START; } else { yypushback(1); yybegin(CODE_BLOCK); }}
+
   "```" {IDENTIFIER}?  { yybegin(LANG_ID); if (isCodeStart == true) { isCodeStart = false; return CODE_BLOCK_END; } else { isCodeStart = true; }; yypushback(yylength()); }
 
   {TEXT_SEGMENT}       { if(isCodeStart) { return codeContent(); } else { return TEXT_SEGMENT; } }

exts/devin-lang/src/test/kotlin/cc/unitmesh/language/DevInParsingTest.kt

@@ -19,4 +19,8 @@ class DevInParsingTest : ParsingTestCase("parser", "devin", DevInParserDefinitio
     fun testEmptyCodeFence() {
         doTest(true)
     }
+
+    fun testJavaAnnotation() {
+        doTest(true)
+    }
 }

exts/devin-lang/src/test/testData/parser/EmptyCodeFence.txt

@@ -5,7 +5,7 @@ DevInFile
   CodeBlockElement(CODE)
     PsiElement(DevInTokenType.CODE_BLOCK_START)('```')
     PsiElement(DevInTokenType.NEWLINE)('\n')
-    DevInCodeContentsImpl(CODE_CONTENTS)
+    ASTWrapperPsiElement(CODE_CONTENTS)
       PsiElement(DevInTokenType.CODE_CONTENT)('print("Hello, world!")')
       PsiElement(DevInTokenType.NEWLINE)('\n')
     PsiElement(DevInTokenType.CODE_BLOCK_END)('```')

exts/devin-lang/src/test/testData/parser/JavaAnnotation.devin

@@ -0,0 +1,7 @@
+```java
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+public @interface ExampleAnnotation {
+    String value() default "";
+}
+```

exts/devin-lang/src/test/testData/parser/JavaAnnotation.txt

@@ -0,0 +1,18 @@
+DevInFile
+  CodeBlockElement(CODE)
+    PsiElement(DevInTokenType.CODE_BLOCK_START)('```')
+    PsiElement(DevInTokenType.LANGUAGE_ID)('java')
+    PsiElement(DevInTokenType.NEWLINE)('\n')
+    ASTWrapperPsiElement(CODE_CONTENTS)
+      PsiElement(DevInTokenType.CODE_CONTENT)('@Target({ElementType.TYPE})')
+      PsiElement(DevInTokenType.NEWLINE)('\n')
+      PsiElement(DevInTokenType.CODE_CONTENT)('@Retention(RetentionPolicy.RUNTIME)')
+      PsiElement(DevInTokenType.NEWLINE)('\n')
+      PsiElement(DevInTokenType.CODE_CONTENT)('public ')
+      PsiElement(DevInTokenType.CODE_CONTENT)('@interface ExampleAnnotation {')
+      PsiElement(DevInTokenType.NEWLINE)('\n')
+      PsiElement(DevInTokenType.CODE_CONTENT)('    String value() default "";')
+      PsiElement(DevInTokenType.NEWLINE)('\n')
+      PsiElement(DevInTokenType.CODE_CONTENT)('}')
+      PsiElement(DevInTokenType.NEWLINE)('\n')
+    PsiElement(DevInTokenType.CODE_BLOCK_END)('```')

exts/devin-lang/src/test/testData/parser/JavaHelloWorld.txt

@@ -6,7 +6,7 @@ DevInFile
     PsiElement(DevInTokenType.CODE_BLOCK_START)('```')
     PsiElement(DevInTokenType.LANGUAGE_ID)('java')
     PsiElement(DevInTokenType.NEWLINE)('\n')
-    DevInCodeContentsImpl(CODE_CONTENTS)
+    ASTWrapperPsiElement(CODE_CONTENTS)
       PsiElement(DevInTokenType.CODE_CONTENT)('public class Main {')
       PsiElement(DevInTokenType.NEWLINE)('\n')
       PsiElement(DevInTokenType.CODE_CONTENT)('    public static void main(String[] args) {')