feat(devin-lang): init basic code fence support #101

Author: phodal

exts/devin-lang/src/grammar/DevInLexer.flex

@@ -25,33 +25,51 @@ import com.intellij.psi.TokenType;
 %s AGENT_BLOCK
 %s VARIABLE_BLOCK
 %s COMMAND_BLOCK
+%s CODE_BLOCK
 
 IDENTIFIER=[a-zA-Z0-9][_\-a-zA-Z0-9]*
 VARIABLE_ID=[a-zA-Z0-9][_\-a-zA-Z0-9]*
 AGENT_ID=[a-zA-Z0-9][_\-a-zA-Z0-9]*
 COMMAND_ID=[a-zA-Z0-9][_\-a-zA-Z0-9]*
 REF_BLOCK=([$/@] {IDENTIFIER} )
 TEXT_SEGMENT=[^$/@\n]+
-NEWLINE=\n|\r\n
+//CODE_CONTENT="```" {IDENTIFIER} ([^$/@\n]+ | \n)* "```"
+CODE_CONTENT=([^$/@\n]+ )
+NEWLINE= \n | \r | \r\n
 
 %{
-    private IElementType contextBlock() {
+    private boolean isCodeStart = false;
+%}
+
+%{
+    private IElementType codeContent() {
         yybegin(YYINITIAL);
 
-        String text = yytext().toString();
+        // handle for end which is \n```
+        String text = yytext().toString().trim();
+        if (text.equals("\n```") || text.equals("```")) {
+            isCodeStart = false;
+            return CODE_BLOCK_END;
+        }
 
-        return TEXT_SEGMENT;
+        // new line
+        if (text.equals("\n")) {
+            return NEWLINE;
+        }
+
+        return CODE_CONTENT;
     }
 %}
 
 %%
 <YYINITIAL> {
-  "@"                  { yybegin(AGENT_BLOCK); return AGENT_START; }
-  "/"                  { yybegin(COMMAND_BLOCK); return COMMAND_START; }
+  "@"                  { yybegin(AGENT_BLOCK);    return AGENT_START; }
+  "/"                  { yybegin(COMMAND_BLOCK);  return COMMAND_START; }
   "$"                  { yybegin(VARIABLE_BLOCK); return VARIABLE_START; }
+  "```" {IDENTIFIER}   { yybegin(CODE_BLOCK); isCodeStart = true; return CODE_BLOCK_START; }
 
-  {TEXT_SEGMENT}       { return TEXT_SEGMENT; }
-  {NEWLINE}            { return NEWLINE; }
+  {TEXT_SEGMENT}       { if(isCodeStart) { return codeContent(); } else { return TEXT_SEGMENT; } }
+  {NEWLINE}            { return NEWLINE;  }
   [^]                  { return TokenType.BAD_CHARACTER; }
 }
 
@@ -69,3 +87,9 @@ NEWLINE=\n|\r\n
   {VARIABLE_ID}        { yybegin(YYINITIAL); return VARIABLE_ID; }
   [^]                  { return TokenType.BAD_CHARACTER; }
 }
+
+<CODE_BLOCK> {
+  {CODE_CONTENT}       { return codeContent(); }
+  <<EOF>>              { isCodeStart = false; return codeContent(); }
+  [^]                  { }
+}

exts/devin-lang/src/grammar/DevInParser.bnf

@@ -15,10 +15,14 @@
   tokenTypeClass="cc.unitmesh.language.lexer.DevInTokenType"
 
   tokens=[
-    AGENT_START    = "AGENT_START"
-    COMMAND_START  = "COMMAND_START"
-    VARIABLE_START = "VARIABLE_START"
-    IDENTIFIER     = "IDENTIFIER"
+    AGENT_START        = "AGENT_START"
+    COMMAND_START      = "COMMAND_START"
+    VARIABLE_START     = "VARIABLE_START"
+    CODE_BLOCK_START   = "CODE_BLOCK_START"
+    CODE_BLOCK_END     = "CODE_BLOCK_END"
+    CODE_BLOCK         = "CODE_BLOCK"
+    CODE_CONTENT       = "CODE_CONTENT"
+    IDENTIFIER         = "IDENTIFIER"
   ]
 }
 
@@ -31,5 +35,5 @@ used ::= (
 )
 
 code ::= (
-   "```" IDENTIFIER TEXT_SEGMENT "```"
+   CODE_BLOCK_START (NEWLINE | CODE_CONTENT)* CODE_BLOCK_END
 )

exts/devin-lang/src/test/kotlin/cc/unitmesh/language/DevInParsingTest.kt

@@ -14,4 +14,8 @@ class DevInParsingTest : ParsingTestCase("parser", "devin", DevInParserDefinitio
     fun testJavaHelloWorld() {
         doTest(true)
     }
-}
+
+    fun testEmptyCodeFence() {
+        doTest(true)
+    }
+}

exts/devin-lang/src/test/testData/parser/EmptyCodeFence.devin

@@ -0,0 +1,7 @@
+解释如下的代码：
+
+```
+print("Hello, world!")
+```
+
+请使用 Markdown 语法返回。

exts/devin-lang/src/test/testData/parser/EmptyCodeFence.txt

@@ -0,0 +1,12 @@
+DevInFile
+  PsiElement(DevInTokenType.TEXT_SEGMENT)('解释如下的代码：')
+  PsiElement(DevInTokenType.NEWLINE)('\n')
+  PsiElement(DevInTokenType.NEWLINE)('\n')
+  PsiElement(DevInTokenType.TEXT_SEGMENT)('```')
+  PsiElement(DevInTokenType.NEWLINE)('\n')
+  PsiElement(DevInTokenType.TEXT_SEGMENT)('print("Hello, world!")')
+  PsiElement(DevInTokenType.NEWLINE)('\n')
+  PsiElement(DevInTokenType.TEXT_SEGMENT)('```')
+  PsiElement(DevInTokenType.NEWLINE)('\n')
+  PsiElement(DevInTokenType.NEWLINE)('\n')
+  PsiElement(DevInTokenType.TEXT_SEGMENT)('请使用 Markdown 语法返回。')

exts/devin-lang/src/test/testData/parser/JavaHelloWorld.devin

@@ -1,7 +1,11 @@
+解释如下的代码：
+
 ```java
 public class Main {
     public static void main(String[] args) {
         System.out.println("Hello, world!");
     }
 }
 ```
+
+请使用 Markdown 语法返回。

exts/devin-lang/src/test/testData/parser/JavaHelloWorld.txt

@@ -1,14 +1,20 @@
 DevInFile
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('```java')
+  PsiElement(DevInTokenType.TEXT_SEGMENT)('解释如下的代码：')
   PsiElement(DevInTokenType.NEWLINE)('\n')
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('public class Main {')
   PsiElement(DevInTokenType.NEWLINE)('\n')
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('    public static void main(String[] args) {')
+  DevInCodeImpl(CODE)
+    PsiElement(DevInTokenType.CODE_BLOCK_START)('```java')
+    PsiElement(DevInTokenType.CODE_CONTENT)('\npublic class Main {')
+    PsiElement(DevInTokenType.NEWLINE)('\n')
+    PsiElement(DevInTokenType.CODE_CONTENT)('    public static void main(String[] args) {')
+    PsiElement(DevInTokenType.NEWLINE)('\n')
+    PsiElement(DevInTokenType.CODE_CONTENT)('        System.out.println("Hello, world!");')
+    PsiElement(DevInTokenType.NEWLINE)('\n')
+    PsiElement(DevInTokenType.CODE_CONTENT)('    }')
+    PsiElement(DevInTokenType.NEWLINE)('\n')
+    PsiElement(DevInTokenType.CODE_CONTENT)('}')
+    PsiElement(DevInTokenType.NEWLINE)('\n')
+    PsiElement(DevInTokenType.CODE_BLOCK_END)('```')
   PsiElement(DevInTokenType.NEWLINE)('\n')
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('        System.out.println("Hello, world!");')
   PsiElement(DevInTokenType.NEWLINE)('\n')
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('    }')
-  PsiElement(DevInTokenType.NEWLINE)('\n')
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('}')
-  PsiElement(DevInTokenType.NEWLINE)('\n')
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('```')
+  PsiElement(DevInTokenType.TEXT_SEGMENT)('请使用 Markdown 语法返回。')