fix(devin-lang): update parser and lexer tokens and rules for improved grammar and tokenization. #101

Author: phodal

exts/devin-lang/src/grammar/DevInLexer.flex

@@ -27,7 +27,7 @@ import com.intellij.psi.TokenType;
 
 IDENTIFIER=[a-zA-Z0-9]([_\-a-zA-Z0-9]*)
 REF_BLOCK=("@" {IDENTIFIER} )
-TEXT_SEGMENT=.*
+TEXT_SEGMENT=[^$/@]+
 NEWLINE=\n|\r\n
 
 %{

exts/devin-lang/src/grammar/DevInParser.bnf

@@ -13,6 +13,11 @@
   elementTypeHolderClass="cc.unitmesh.language.psi.DevInTypes"
   elementTypeClass="cc.unitmesh.language.psi.DevInElementType"
   tokenTypeClass="cc.unitmesh.language.lexer.DevInTokenType"
+
+  tokens=[
+  ]
 }
 
-DevInFile ::= (REF_BLOCK | TEXT_SEGMENT | NEWLINE)*
+DevInFile ::= (REF_BLOCK | TEXT_SEGMENT | CODE_FENCE | NEWLINE)*
+
+//used ::= (AGENT | COMMAND | VARIABLE) IDENTIFIER

exts/devin-lang/src/test/testData/parser/BasicTest.txt

@@ -1,2 +1,4 @@
 FILE
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('你好 @hello-world sm')
+  PsiElement(DevInTokenType.TEXT_SEGMENT)('你好 ')
+  PsiElement(DevInTokenType.REF_BLOCK)('@hello-world')
+  PsiElement(DevInTokenType.TEXT_SEGMENT)(' sm')

exts/devin-lang/src/test/testData/parser/JavaHelloWorld.txt

@@ -1,14 +1,2 @@
 FILE
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('```java')
-  PsiElement(DevInTokenType.NEWLINE)('\n')
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('public class Main {')
-  PsiElement(DevInTokenType.NEWLINE)('\n')
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('    public static void main(String[] args) {')
-  PsiElement(DevInTokenType.NEWLINE)('\n')
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('        System.out.println("Hello, world!");')
-  PsiElement(DevInTokenType.NEWLINE)('\n')
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('    }')
-  PsiElement(DevInTokenType.NEWLINE)('\n')
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('}')
-  PsiElement(DevInTokenType.NEWLINE)('\n')
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('```')
+  PsiElement(DevInTokenType.TEXT_SEGMENT)('```java\npublic class Main {\n    public static void main(String[] args) {\n        System.out.println("Hello, world!");\n    }\n}\n```')