---

yaml
---
r: 205822
b: refs/heads/beta
c: dd24070
h: refs/heads/master
v: v3

Author: pnkfelix

[refs]

@@ -29,7 +29,7 @@ refs/tags/0.12.0: f0c419429ef30723ceaf6b42f9b5a2aeb5d2e2d1
 refs/heads/automation-fail: 1bf06495443584539b958873e04cc2f864ab10e4
 refs/heads/batch: b7fd822592a4fb577552d93010c4a4e14f314346
 refs/heads/building: 126db549b038c84269a1e4fe46f051b2c15d6970
-refs/heads/beta: 7397bdc9c516f3f714ad4974ecdd27f567d03d05
+refs/heads/beta: dd240707999216a64e7914c2290cb59c54d9c60c
 refs/heads/windistfix: 7608dbad651f02e837ed05eef3d74a6662a6e928
 refs/tags/1.0.0-alpha: e42bd6d93a1d3433c486200587f8f9e12590a4d7
 refs/heads/tmp: 579e31929feff51dcaf8d444648eff8de735f91a

branches/beta/src/grammar/README.md

@@ -12,7 +12,7 @@ javac *.java
 rustc -O verify.rs
 for file in ../*/**.rs; do
     echo $file;
-    grun RustLexer tokens -tokens < "$file" | ./verify "$file" RustLexer.tokens || break
+    grun RustLexer tokens -tokens < $file | ./verify $file RustLexer.tokens || break
 done
 ```
 

branches/beta/src/grammar/RustLexer.g4

@@ -1,12 +1,5 @@
 lexer grammar RustLexer;
 
-@lexer::members {
-  public boolean is_at(int pos) {
-    return _input.index() == pos;
-  }
-}
-
-
 tokens {
     EQ, LT, LE, EQEQ, NE, GE, GT, ANDAND, OROR, NOT, TILDE, PLUT,
     MINUS, STAR, SLASH, PERCENT, CARET, AND, OR, SHL, SHR, BINOP,
@@ -15,10 +8,14 @@ tokens {
     LBRACE, RBRACE, POUND, DOLLAR, UNDERSCORE, LIT_CHAR,
     LIT_INTEGER, LIT_FLOAT, LIT_STR, LIT_STR_RAW, LIT_BINARY,
     LIT_BINARY_RAW, IDENT, LIFETIME, WHITESPACE, DOC_COMMENT,
-    COMMENT, SHEBANG
+    COMMENT
 }
 
-import xidstart , xidcontinue;
+/* Note: due to antlr limitations, we can't represent XID_start and
+ * XID_continue properly. ASCII-only substitute. */
+
+fragment XID_start : [_a-zA-Z] ;
+fragment XID_continue : [_a-zA-Z0-9] ;
 
 
 /* Expression-operator symbols */
@@ -93,63 +90,94 @@ fragment CHAR_ESCAPE
   | [xX] HEXIT HEXIT
   | 'u' HEXIT HEXIT HEXIT HEXIT
   | 'U' HEXIT HEXIT HEXIT HEXIT HEXIT HEXIT HEXIT HEXIT
-  | 'u{' HEXIT '}'
-  | 'u{' HEXIT HEXIT '}'
-  | 'u{' HEXIT HEXIT HEXIT '}'
-  | 'u{' HEXIT HEXIT HEXIT HEXIT '}'
-  | 'u{' HEXIT HEXIT HEXIT HEXIT HEXIT '}'
-  | 'u{' HEXIT HEXIT HEXIT HEXIT HEXIT HEXIT '}'
   ;
 
 fragment SUFFIX
   : IDENT
   ;
 
-fragment INTEGER_SUFFIX
-  : { _input.LA(1) != 'e' && _input.LA(1) != 'E' }? SUFFIX
-  ;
-
 LIT_CHAR
-  : '\'' ( '\\' CHAR_ESCAPE
-         | ~[\\'\n\t\r]
-         | '\ud800' .. '\udbff' '\udc00' .. '\udfff'
-         )
-    '\'' SUFFIX?
+  : '\'' ( '\\' CHAR_ESCAPE | ~[\\'\n\t\r] ) '\'' SUFFIX?
   ;
 
 LIT_BYTE
-  : 'b\'' ( '\\' ( [xX] HEXIT HEXIT
-                 | [nrt\\'"0] )
-          | ~[\\'\n\t\r] '\udc00'..'\udfff'?
-          )
-    '\'' SUFFIX?
+  : 'b\'' ( '\\' ( [xX] HEXIT HEXIT | [nrt\\'"0] ) | ~[\\'\n\t\r] ) '\'' SUFFIX?
   ;
 
 LIT_INTEGER
-
-  : [0-9][0-9_]* INTEGER_SUFFIX?
-  | '0b' [01_]+ INTEGER_SUFFIX?
-  | '0o' [0-7_]+ INTEGER_SUFFIX?
-  | '0x' [0-9a-fA-F_]+ INTEGER_SUFFIX?
+  : [0-9][0-9_]* SUFFIX?
+  | '0b' [01][01_]* SUFFIX?
+  | '0o' [0-7][0-7_]* SUFFIX?
+  | '0x' [0-9a-fA-F][0-9a-fA-F_]* SUFFIX?
   ;
 
 LIT_FLOAT
   : [0-9][0-9_]* ('.' {
-        /* dot followed by another dot is a range, not a float */
+        /* dot followed by another dot is a range, no float */
         _input.LA(1) != '.' &&
-        /* dot followed by an identifier is an integer with a function call, not a float */
+        /* dot followed by an identifier is an integer with a function call, no float */
         _input.LA(1) != '_' &&
-        !(_input.LA(1) >= 'a' && _input.LA(1) <= 'z') &&
-        !(_input.LA(1) >= 'A' && _input.LA(1) <= 'Z')
+        _input.LA(1) != 'a' &&
+        _input.LA(1) != 'b' &&
+        _input.LA(1) != 'c' &&
+        _input.LA(1) != 'd' &&
+        _input.LA(1) != 'e' &&
+        _input.LA(1) != 'f' &&
+        _input.LA(1) != 'g' &&
+        _input.LA(1) != 'h' &&
+        _input.LA(1) != 'i' &&
+        _input.LA(1) != 'j' &&
+        _input.LA(1) != 'k' &&
+        _input.LA(1) != 'l' &&
+        _input.LA(1) != 'm' &&
+        _input.LA(1) != 'n' &&
+        _input.LA(1) != 'o' &&
+        _input.LA(1) != 'p' &&
+        _input.LA(1) != 'q' &&
+        _input.LA(1) != 'r' &&
+        _input.LA(1) != 's' &&
+        _input.LA(1) != 't' &&
+        _input.LA(1) != 'u' &&
+        _input.LA(1) != 'v' &&
+        _input.LA(1) != 'w' &&
+        _input.LA(1) != 'x' &&
+        _input.LA(1) != 'y' &&
+        _input.LA(1) != 'z' &&
+        _input.LA(1) != 'A' &&
+        _input.LA(1) != 'B' &&
+        _input.LA(1) != 'C' &&
+        _input.LA(1) != 'D' &&
+        _input.LA(1) != 'E' &&
+        _input.LA(1) != 'F' &&
+        _input.LA(1) != 'G' &&
+        _input.LA(1) != 'H' &&
+        _input.LA(1) != 'I' &&
+        _input.LA(1) != 'J' &&
+        _input.LA(1) != 'K' &&
+        _input.LA(1) != 'L' &&
+        _input.LA(1) != 'M' &&
+        _input.LA(1) != 'N' &&
+        _input.LA(1) != 'O' &&
+        _input.LA(1) != 'P' &&
+        _input.LA(1) != 'Q' &&
+        _input.LA(1) != 'R' &&
+        _input.LA(1) != 'S' &&
+        _input.LA(1) != 'T' &&
+        _input.LA(1) != 'U' &&
+        _input.LA(1) != 'V' &&
+        _input.LA(1) != 'W' &&
+        _input.LA(1) != 'X' &&
+        _input.LA(1) != 'Y' &&
+        _input.LA(1) != 'Z'
   }? | ('.' [0-9][0-9_]*)? ([eE] [-+]? [0-9][0-9_]*)? SUFFIX?)
   ;
 
 LIT_STR
   : '"' ('\\\n' | '\\\r\n' | '\\' CHAR_ESCAPE | .)*? '"' SUFFIX?
   ;
 
-LIT_BINARY : 'b' LIT_STR ;
-LIT_BINARY_RAW : 'b' LIT_STR_RAW ;
+LIT_BINARY : 'b' LIT_STR SUFFIX?;
+LIT_BINARY_RAW : 'rb' LIT_STR_RAW SUFFIX?;
 
 /* this is a bit messy */
 
@@ -169,27 +197,21 @@ LIT_STR_RAW
 
 QUESTION : '?';
 
-IDENT : XID_Start XID_Continue* ;
+IDENT : XID_start XID_continue* ;
 
 fragment QUESTION_IDENTIFIER : QUESTION? IDENT;
 
 LIFETIME : '\'' IDENT ;
 
 WHITESPACE : [ \r\n\t]+ ;
 
-UNDOC_COMMENT     : '////' ~[\n]* -> type(COMMENT) ;
+UNDOC_COMMENT     : '////' ~[\r\n]* -> type(COMMENT) ;
 YESDOC_COMMENT    : '///' ~[\r\n]* -> type(DOC_COMMENT) ;
 OUTER_DOC_COMMENT : '//!' ~[\r\n]* -> type(DOC_COMMENT) ;
-LINE_COMMENT      : '//' ( ~[/\n] ~[\n]* )? -> type(COMMENT) ;
+LINE_COMMENT      : '//' ~[\r\n]* -> type(COMMENT) ;
 
 DOC_BLOCK_COMMENT
   : ('/**' ~[*] | '/*!') (DOC_BLOCK_COMMENT | .)*? '*/' -> type(DOC_COMMENT)
   ;
 
 BLOCK_COMMENT : '/*' (BLOCK_COMMENT | .)*? '*/' -> type(COMMENT) ;
-
-/* these appear at the beginning of a file */
-
-SHEBANG : '#!' { is_at(2) && _input.LA(1) != '[' }? ~[\r\n]* -> type(SHEBANG) ;
-
-UTF8_BOM : '\ufeff' { is_at(1) }? -> skip ;

branches/beta/src/grammar/check.sh

@@ -18,13 +18,13 @@ failed=0
 skipped=0
 
 check() {
-    grep --silent "// ignore-lexer-test" "$1";
+    grep --silent "// ignore-lexer-test" $1;
 
     # if it's *not* found...
     if [ $? -eq 1 ]; then
         cd $2 # This `cd` is so java will pick up RustLexer.class. I couldn't
-        # figure out how to wrangle the CLASSPATH, just adding build/grammar
-        # didn't seem to have any effect.
+        # figure out how to wrangle the CLASSPATH, just adding build/grammr didn't
+        # seem to have anny effect.
         if $3 RustLexer tokens -tokens < $1 | $4 $1 $5; then
             echo "pass: $1"
             passed=`expr $passed + 1`
@@ -39,7 +39,7 @@ check() {
 }
 
 for file in $(find $1 -iname '*.rs' ! -path '*/test/compile-fail*'); do
-    check "$file" $2 $3 $4 $5
+    check $file $2 $3 $4 $5
 done
 
 printf "\ntest result: "