feat(devin-lang): update code_content support #101

Author: phodal

exts/devin-lang/src/grammar/DevInLexer.flex

@@ -26,14 +26,16 @@ import com.intellij.psi.TokenType;
 %s VARIABLE_BLOCK
 %s COMMAND_BLOCK
 %s CODE_BLOCK
+%s LANG_ID
 
 IDENTIFIER=[a-zA-Z0-9][_\-a-zA-Z0-9]*
+
 VARIABLE_ID=[a-zA-Z0-9][_\-a-zA-Z0-9]*
 AGENT_ID=[a-zA-Z0-9][_\-a-zA-Z0-9]*
 COMMAND_ID=[a-zA-Z0-9][_\-a-zA-Z0-9]*
-REF_BLOCK=([$/@] {IDENTIFIER} )
+LANGUAGE_ID=[a-zA-Z0-9][_\-a-zA-Z0-9]*
+
 TEXT_SEGMENT=[^$/@\n]+
-//CODE_CONTENT="```" {IDENTIFIER} ([^$/@\n]+ | \n)* "```"
 CODE_CONTENT=([^$/@\n]+ )
 NEWLINE= \n | \r | \r\n
 
@@ -66,7 +68,7 @@ NEWLINE= \n | \r | \r\n
   "@"                  { yybegin(AGENT_BLOCK);    return AGENT_START; }
   "/"                  { yybegin(COMMAND_BLOCK);  return COMMAND_START; }
   "$"                  { yybegin(VARIABLE_BLOCK); return VARIABLE_START; }
-  "```" {IDENTIFIER}   { yybegin(CODE_BLOCK); isCodeStart = true; return CODE_BLOCK_START; }
+  "```" {IDENTIFIER}   { yybegin(LANG_ID); isCodeStart = true; yypushback(yylength()); }
 
   {TEXT_SEGMENT}       { if(isCodeStart) { return codeContent(); } else { return TEXT_SEGMENT; } }
   {NEWLINE}            { return NEWLINE;  }
@@ -90,6 +92,13 @@ NEWLINE= \n | \r | \r\n
 
 <CODE_BLOCK> {
   {CODE_CONTENT}       { return codeContent(); }
+  {NEWLINE}            { return NEWLINE; }
   <<EOF>>              { isCodeStart = false; return codeContent(); }
   [^]                  { }
 }
+
+<LANG_ID> {
+   "```"             { return CODE_BLOCK_START; }
+   {IDENTIFIER}      { yybegin(YYINITIAL); return LANGUAGE_ID;  }
+   [^]               { yypushback(1); yybegin(CODE_BLOCK); }
+}

exts/devin-lang/src/grammar/DevInParser.bnf

@@ -23,6 +23,7 @@
     CODE_BLOCK         = "CODE_BLOCK"
     CODE_CONTENT       = "CODE_CONTENT"
     IDENTIFIER         = "IDENTIFIER"
+    LANGUAGE_ID        = "LANGUAGE_ID"
   ]
 }
 
@@ -35,5 +36,7 @@ used ::= (
 )
 
 code ::= (
-   CODE_BLOCK_START (NEWLINE | CODE_CONTENT)* CODE_BLOCK_END
-)
+   CODE_BLOCK_START LANGUAGE_ID code_contents CODE_BLOCK_END
+)
+
+code_contents ::= (NEWLINE | CODE_CONTENT)*

exts/devin-lang/src/test/testData/parser/JavaHelloWorld.txt

@@ -3,17 +3,20 @@ DevInFile
   PsiElement(DevInTokenType.NEWLINE)('\n')
   PsiElement(DevInTokenType.NEWLINE)('\n')
   DevInCodeImpl(CODE)
-    PsiElement(DevInTokenType.CODE_BLOCK_START)('```java')
-    PsiElement(DevInTokenType.CODE_CONTENT)('\npublic class Main {')
-    PsiElement(DevInTokenType.NEWLINE)('\n')
-    PsiElement(DevInTokenType.CODE_CONTENT)('    public static void main(String[] args) {')
-    PsiElement(DevInTokenType.NEWLINE)('\n')
-    PsiElement(DevInTokenType.CODE_CONTENT)('        System.out.println("Hello, world!");')
-    PsiElement(DevInTokenType.NEWLINE)('\n')
-    PsiElement(DevInTokenType.CODE_CONTENT)('    }')
-    PsiElement(DevInTokenType.NEWLINE)('\n')
-    PsiElement(DevInTokenType.CODE_CONTENT)('}')
-    PsiElement(DevInTokenType.NEWLINE)('\n')
+    PsiElement(DevInTokenType.CODE_BLOCK_START)('```')
+    PsiElement(DevInTokenType.LANGUAGE_ID)('java')
+    DevInCodeContentsImpl(CODE_CONTENTS)
+      PsiElement(DevInTokenType.NEWLINE)('\n')
+      PsiElement(DevInTokenType.CODE_CONTENT)('public class Main {')
+      PsiElement(DevInTokenType.NEWLINE)('\n')
+      PsiElement(DevInTokenType.CODE_CONTENT)('    public static void main(String[] args) {')
+      PsiElement(DevInTokenType.NEWLINE)('\n')
+      PsiElement(DevInTokenType.CODE_CONTENT)('        System.out.println("Hello, world!");')
+      PsiElement(DevInTokenType.NEWLINE)('\n')
+      PsiElement(DevInTokenType.CODE_CONTENT)('    }')
+      PsiElement(DevInTokenType.NEWLINE)('\n')
+      PsiElement(DevInTokenType.CODE_CONTENT)('}')
+      PsiElement(DevInTokenType.NEWLINE)('\n')
     PsiElement(DevInTokenType.CODE_BLOCK_END)('```')
   PsiElement(DevInTokenType.NEWLINE)('\n')
   PsiElement(DevInTokenType.NEWLINE)('\n')