refactor(devin-lang): split \n to new language #101

Author: phodal

exts/devin-lang/src/grammar/DevInLexer.flex

@@ -31,7 +31,7 @@ VARIABLE_ID=[a-zA-Z0-9][_\-a-zA-Z0-9]*
 AGENT_ID=[a-zA-Z0-9][_\-a-zA-Z0-9]*
 COMMAND_ID=[a-zA-Z0-9][_\-a-zA-Z0-9]*
 REF_BLOCK=([$/@] {IDENTIFIER} )
-TEXT_SEGMENT=[^$/@]+
+TEXT_SEGMENT=[^$/@\n]+
 NEWLINE=\n|\r\n
 
 %{

exts/devin-lang/src/grammar/DevInParser.bnf

@@ -22,10 +22,14 @@
   ]
 }
 
-DevInFile ::= (used | TEXT_SEGMENT | CODE_FENCE | NEWLINE)*
+DevInFile ::= (used | code | TEXT_SEGMENT | NEWLINE)*
 
 used ::= (
     AGENT_START AGENT_ID?
     | COMMAND_START COMMAND_ID?
     | VARIABLE_START VARIABLE_ID?
 )
+
+code ::= (
+   "```" IDENTIFIER TEXT_SEGMENT "```"
+)

exts/devin-lang/src/test/testData/parser/BasicTest.txt

@@ -3,11 +3,13 @@ DevInFile
   DevInUsedImpl(USED)
     PsiElement(DevInTokenType.AGENT_START)('@')
     PsiElement(DevInTokenType.AGENT_ID)('hello-world')
-  PsiElement(DevInTokenType.TEXT_SEGMENT)(' sm\n解释一下代码 ')
+  PsiElement(DevInTokenType.TEXT_SEGMENT)(' sm')
+  PsiElement(DevInTokenType.NEWLINE)('\n')
+  PsiElement(DevInTokenType.TEXT_SEGMENT)('解释一下代码 ')
   DevInUsedImpl(USED)
     PsiElement(DevInTokenType.VARIABLE_START)('$')
     PsiElement(DevInTokenType.VARIABLE_ID)('selection')
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('\n')
+  PsiElement(DevInTokenType.NEWLINE)('\n')
   DevInUsedImpl(USED)
     PsiElement(DevInTokenType.COMMAND_START)('/')
     PsiElement(DevInTokenType.COMMAND_ID)('agent')

exts/devin-lang/src/test/testData/parser/JavaHelloWorld.txt

@@ -1,2 +1,14 @@
 DevInFile
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('```java\npublic class Main {\n    public static void main(String[] args) {\n        System.out.println("Hello, world!");\n    }\n}\n```')
+  PsiElement(DevInTokenType.TEXT_SEGMENT)('```java')
+  PsiElement(DevInTokenType.NEWLINE)('\n')
+  PsiElement(DevInTokenType.TEXT_SEGMENT)('public class Main {')
+  PsiElement(DevInTokenType.NEWLINE)('\n')
+  PsiElement(DevInTokenType.TEXT_SEGMENT)('    public static void main(String[] args) {')
+  PsiElement(DevInTokenType.NEWLINE)('\n')
+  PsiElement(DevInTokenType.TEXT_SEGMENT)('        System.out.println("Hello, world!");')
+  PsiElement(DevInTokenType.NEWLINE)('\n')
+  PsiElement(DevInTokenType.TEXT_SEGMENT)('    }')
+  PsiElement(DevInTokenType.NEWLINE)('\n')
+  PsiElement(DevInTokenType.TEXT_SEGMENT)('}')
+  PsiElement(DevInTokenType.NEWLINE)('\n')
+  PsiElement(DevInTokenType.TEXT_SEGMENT)('```')