feat(devin-lang): update language grammar and lexer to support optional language identifiers in code blocks. #101

Author: phodal

exts/devin-lang/src/grammar/DevInLexer.flex

@@ -49,7 +49,7 @@ NEWLINE= \n | \r | \r\n
 
         // handle for end which is \n```
         String text = yytext().toString().trim();
-        if (text.equals("\n```") || text.equals("```")) {
+        if ((text.equals("\n```") || text.equals("```")) && isCodeStart == true ) {
             isCodeStart = false;
             return CODE_BLOCK_END;
         }
@@ -59,6 +59,10 @@ NEWLINE= \n | \r | \r\n
             return NEWLINE;
         }
 
+        if (isCodeStart == false) {
+            return TEXT_SEGMENT;
+        }
+
         return CODE_CONTENT;
     }
 %}
@@ -68,7 +72,7 @@ NEWLINE= \n | \r | \r\n
   "@"                  { yybegin(AGENT_BLOCK);    return AGENT_START; }
   "/"                  { yybegin(COMMAND_BLOCK);  return COMMAND_START; }
   "$"                  { yybegin(VARIABLE_BLOCK); return VARIABLE_START; }
-  "```" {IDENTIFIER}   { yybegin(LANG_ID); isCodeStart = true; yypushback(yylength()); }
+  "```" {IDENTIFIER}?  { yybegin(LANG_ID); if (isCodeStart == true) { isCodeStart = false; return CODE_BLOCK_END; } else { isCodeStart = true; }; yypushback(yylength()); }
 
   {TEXT_SEGMENT}       { if(isCodeStart) { return codeContent(); } else { return TEXT_SEGMENT; } }
   {NEWLINE}            { return NEWLINE;  }
@@ -91,10 +95,10 @@ NEWLINE= \n | \r | \r\n
 }
 
 <CODE_BLOCK> {
-  {CODE_CONTENT}       { return codeContent(); }
+  {CODE_CONTENT}       { if(isCodeStart) { return codeContent(); } else { yybegin(YYINITIAL); yypushback(yylength()); } }
   {NEWLINE}            { return NEWLINE; }
-  <<EOF>>              { isCodeStart = false; return codeContent(); }
-  [^]                  { }
+  <<EOF>>              { isCodeStart = false; yybegin(YYINITIAL); yypushback(yylength()); }
+  [^]                  { isCodeStart = false; yybegin(YYINITIAL); yypushback(yylength()); }
 }
 
 <LANG_ID> {

exts/devin-lang/src/grammar/DevInParser.bnf

@@ -35,8 +35,6 @@ used ::= (
     | VARIABLE_START VARIABLE_ID?
 )
 
-code ::= (
-   CODE_BLOCK_START LANGUAGE_ID code_contents CODE_BLOCK_END
-)
+code ::=  CODE_BLOCK_START LANGUAGE_ID? code_contents CODE_BLOCK_END
 
 code_contents ::= (NEWLINE | CODE_CONTENT)*

exts/devin-lang/src/test/testData/parser/EmptyCodeFence.txt

@@ -2,11 +2,13 @@ DevInFile
   PsiElement(DevInTokenType.TEXT_SEGMENT)('解释如下的代码：')
   PsiElement(DevInTokenType.NEWLINE)('\n')
   PsiElement(DevInTokenType.NEWLINE)('\n')
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('```')
-  PsiElement(DevInTokenType.NEWLINE)('\n')
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('print("Hello, world!")')
-  PsiElement(DevInTokenType.NEWLINE)('\n')
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('```')
+  DevInCodeImpl(CODE)
+    PsiElement(DevInTokenType.CODE_BLOCK_START)('```')
+    DevInCodeContentsImpl(CODE_CONTENTS)
+      PsiElement(DevInTokenType.NEWLINE)('\n')
+      PsiElement(DevInTokenType.CODE_CONTENT)('print("Hello, world!")')
+      PsiElement(DevInTokenType.NEWLINE)('\n')
+    PsiElement(DevInTokenType.CODE_BLOCK_END)('```')
   PsiElement(DevInTokenType.NEWLINE)('\n')
   PsiElement(DevInTokenType.NEWLINE)('\n')
   PsiElement(DevInTokenType.TEXT_SEGMENT)('请使用 Markdown 语法返回。')