Skip to content

Commit 6beb383

Browse files
committed
fix(lexer): handle special chars in text segments
Modify lexer logic to handle special characters (@, /, $) at the start of text segments by transitioning to appropriate states (AGENT_BLOCK, COMMAND_BLOCK, VARIABLE_BLOCK) and returning corresponding tokens.
1 parent 495a7aa commit 6beb383

File tree

2 files changed

+16
-7
lines changed

2 files changed

+16
-7
lines changed

exts/devins-lang/src/grammar/DevInLexer.flex

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -109,8 +109,21 @@ SHARP=#
109109
return comment();
110110
}
111111

112-
yypushback(yylength());
113-
yybegin(YYUSED);
112+
// get first char, if $, @, /, #, should be a YYUSED
113+
char first = text.charAt(0);
114+
if (first == '@') {
115+
yypushback(yylength() - 1);
116+
yybegin(AGENT_BLOCK);
117+
return AGENT_START;
118+
} else if (first == '/') {
119+
yypushback(yylength() - 1);
120+
yybegin(COMMAND_BLOCK);
121+
return COMMAND_START;
122+
} else if (first == '$') {
123+
yypushback(yylength() - 1);
124+
yybegin(VARIABLE_BLOCK);
125+
return VARIABLE_START;
126+
}
114127

115128
return TEXT_SEGMENT;
116129
}

exts/devins-lang/src/test/testData/parser/BasicTest.txt

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,5 @@
11
DevInFile
2-
PsiElement(DevInTokenType.TEXT_SEGMENT)('你好 ')
3-
DevInUsedImpl(USED)
4-
PsiElement(DevInTokenType.AGENT_START)('@')
5-
PsiElement(DevInTokenType.AGENT_ID)('hello-world')
6-
PsiElement(DevInTokenType.TEXT_SEGMENT)(' sm')
2+
PsiElement(DevInTokenType.TEXT_SEGMENT)('你好 @hello-world sm')
73
PsiElement(DevInTokenType.NEWLINE)('\n')
84
PsiElement(DevInTokenType.TEXT_SEGMENT)('解释一下代码')
95
PsiElement(DevInTokenType.NEWLINE)('\n')

0 commit comments

Comments
 (0)