---

yaml
---
r: 124924
b: refs/heads/auto
c: 76a1552
h: refs/heads/master
v: v3

Author: emberian

[refs]

@@ -13,7 +13,7 @@ refs/heads/try3: 9387340aab40a73e8424c48fd42f0c521a4875c0
 refs/tags/release-0.3.1: 495bae036dfe5ec6ceafd3312b4dca48741e845b
 refs/tags/release-0.4: e828ea2080499553b97dfe33b3f4d472b4562ad7
 refs/tags/release-0.5: 7e3bcfbf21278251ee936ad53e92e9b719702d73
-refs/heads/auto: 1a1a9d54456355cc0ebdd397fd04871abe27f78c
+refs/heads/auto: 76a15520212b1b9a72c099e0a163a31c102fcde4
 refs/heads/servo: af82457af293e2a842ba6b7759b70288da276167
 refs/tags/release-0.6: b4ebcfa1812664df5e142f0134a5faea3918544c
 refs/tags/0.1: b19db808c2793fe2976759b85a355c3ad8c8b336

branches/auto/src/grammar/RustLexer.g4

@@ -1,11 +1,23 @@
 lexer grammar RustLexer;
 
+tokens {
+    EQ, LT, LE, EQEQ, NE, GE, GT, ANDAND, OROR, NOT, TILDE, PLUT,
+    MINUS, STAR, SLASH, PERCENT, CARET, AND, OR, SHL, SHR, BINOP,
+    BINOPEQ, AT, DOT, DOTDOT, DOTDOTDOT, COMMA, SEMI, COLON,
+    MOD_SEP, RARROW, FAT_ARROW, LPAREN, RPAREN, LBRACKET, RBRACKET,
+    LBRACE, RBRACE, POUND, DOLLAR, UNDERSCORE, LIT_CHAR,
+    LIT_INTEGER, LIT_FLOAT, LIT_STR, LIT_STR_RAW, LIT_BINARY,
+    LIT_BINARY_RAW, IDENT, LIFETIME, WHITESPACE, DOC_COMMENT,
+    COMMENT
+}
+
 /* Note: due to antlr limitations, we can't represent XID_start and
  * XID_continue properly. ASCII-only substitute. */
 
 fragment XID_start : [_a-zA-Z] ;
 fragment XID_continue : [_a-zA-Z0-9] ;
 
+
 /* Expression-operator symbols */
 
 EQ      : '=' ;
@@ -83,7 +95,7 @@ LIT_CHAR
   : '\'' ( '\\' CHAR_ESCAPE | ~[\\'\n\t\r] ) '\''
   ;
 
-INT_SUFFIX
+fragment INT_SUFFIX
   : 'i'
   | 'i8'
   | 'i16'
@@ -141,25 +153,28 @@ LIFETIME : '\'' IDENT ;
 
 WHITESPACE : [ \r\n\t]+ ;
 
-COMMENT
-  : '//' ~[\r\n]*
-  | '////' ~[\r\n]*
-  | BLOCK_COMMENT
-  ;
+LINE_COMMENT_NOT_A_TOKEN : '//' -> more, pushMode(LINE_COMMENT) ;
 
-mode DOCCOMMENT;
 
-fragment DOC_BLOCK_COMMENT
-  : ('/**' | '/*!') (DOC_BLOCK_COMMENT | .)*? '*/'
+DOC_BLOCK_COMMENT
+  : ('/**' | '/*!') (DOC_BLOCK_COMMENT | .)*? '*/' -> type(DOC_COMMENT)
   ;
 
-DOC_COMMENT
-  : '///' ~[\r\n]*
-  | '//!' ~[\r\n]*
-  | DOC_BLOCK_COMMENT
+BLOCK_COMMENT : '/*' (BLOCK_COMMENT | .)*? '*/' -> type(COMMENT) ;
+
+mode LINE_COMMENT;
+
+MAYBE_DOC_COMMENT
+  : '/' -> more, pushMode(LINE_DOC_COMMENT)
   ;
 
-fragment BLOCK_COMMENT
-  : '/*' (BLOCK_COMMENT | .)*? '*/'
+MAYBE_OUTER_DOC_COMMENT
+  : '!' ~[\r\n]* -> type(LINE_DOC_COMMENT), popMode
   ;
 
+COMMENT : ~[\r\n]* -> popMode ;
+
+mode LINE_DOC_COMMENT;
+
+ACTUALLY_A_COMMENT : '/' ~[\r\n]* -> type(COMMENT), popMode ;
+REALLY_A_DOC_COMMENT : ~[\r\n]* -> type(DOC_COMMENT), popMode ;