---

yaml
---
r: 160237
b: refs/heads/snap-stage3
c: 6679595
h: refs/heads/master
i:
  160235: fc42177
v: v3

Author: huonw

[refs]

@@ -1,7 +1,7 @@
 ---
 refs/heads/master: bfaa7bcab3459907014c31d3bf980f65ccd14b08
 refs/heads/snap-stage1: e33de59e47c5076a89eadeb38f4934f58a3618a6
-refs/heads/snap-stage3: ff0278bc152c5e3072434615cf9861e98a160e71
+refs/heads/snap-stage3: 6679595853705ca11f64984a055be60233321a4a
 refs/heads/try: 225de0d60f8ca8dcc62ab2fd8818ebbda4b58cfe
 refs/tags/release-0.1: 1f5c5126e96c79d22cb7862f75304136e204f105
 refs/heads/dist-snap: ba4081a5a8573875fed17545846f6f6902c8ba8d

branches/snap-stage3/src/librustdoc/html/highlight.rs

@@ -128,13 +128,17 @@ fn doit(sess: &parse::ParseSess, mut lexer: lexer::StringReader,
                 }
             }
 
-            // text literals
-            token::Literal(token::Byte(..)) | token::Literal(token::Char(..)) |
-                token::Literal(token::Binary(..)) | token::Literal(token::BinaryRaw(..)) |
-                token::Literal(token::Str_(..)) | token::Literal(token::StrRaw(..)) => "string",
-
-            // number literals
-            token::Literal(token::Integer(..)) | token::Literal(token::Float(..)) => "number",
+            token::Literal(lit, _suf) => {
+                match lit {
+                    // text literals
+                    token::Byte(..) | token::Char(..) |
+                        token::Binary(..) | token::BinaryRaw(..) |
+                        token::Str_(..) | token::StrRaw(..) => "string",
+
+                    // number literals
+                    token::Integer(..) | token::Float(..) => "number",
+                }
+            }
 
             // keywords are also included in the identifier set
             token::Ident(ident, _is_mod_sep) => {

branches/snap-stage3/src/libsyntax/ast.rs

@@ -838,7 +838,7 @@ impl TokenTree {
                     tts: vec![TtToken(sp, token::Ident(token::str_to_ident("doc"),
                                                        token::Plain)),
                               TtToken(sp, token::Eq),
-                              TtToken(sp, token::Literal(token::Str_(name)))],
+                              TtToken(sp, token::Literal(token::Str_(name), None))],
                     close_span: sp,
                 }))
             }

branches/snap-stage3/src/libsyntax/diagnostics/plugin.rs

@@ -87,7 +87,7 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt,
         },
         [ast::TtToken(_, token::Ident(ref code, _)),
          ast::TtToken(_, token::Comma),
-         ast::TtToken(_, token::Literal(token::StrRaw(description, _)))] => {
+         ast::TtToken(_, token::Literal(token::StrRaw(description, _), None))] => {
             (code, Some(description))
         }
         _ => unreachable!()

branches/snap-stage3/src/libsyntax/ext/quote.rs

@@ -543,10 +543,13 @@ fn mk_delim(cx: &ExtCtxt, sp: Span, delim: token::DelimToken) -> P<ast::Expr> {
 #[allow(non_upper_case_globals)]
 fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
     macro_rules! mk_lit {
-        ($name: expr, $($args: expr),*) => {{
+        ($name: expr, $suffix: expr, $($args: expr),*) => {{
             let inner = cx.expr_call(sp, mk_token_path(cx, sp, $name), vec![$($args),*]);
-
-            cx.expr_call(sp, mk_token_path(cx, sp, "Literal"), vec![inner])
+            let suffix = match $suffix {
+                Some(name) => cx.expr_some(sp, mk_name(cx, sp, ast::Ident::new(name))),
+                None => cx.expr_none(sp)
+            };
+            cx.expr_call(sp, mk_token_path(cx, sp, "Literal"), vec![inner, suffix])
         }}
     }
     match *tok {
@@ -567,32 +570,32 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
                                 vec![mk_delim(cx, sp, delim)]);
         }
 
-        token::Literal(token::Byte(i)) => {
+        token::Literal(token::Byte(i), suf) => {
             let e_byte = mk_name(cx, sp, i.ident());
-            return mk_lit!("Byte", e_byte);
+            return mk_lit!("Byte", suf, e_byte);
         }
 
-        token::Literal(token::Char(i)) => {
+        token::Literal(token::Char(i), suf) => {
             let e_char = mk_name(cx, sp, i.ident());
-            return mk_lit!("Char", e_char);
+            return mk_lit!("Char", suf, e_char);
         }
 
-        token::Literal(token::Integer(i)) => {
+        token::Literal(token::Integer(i), suf) => {
             let e_int = mk_name(cx, sp, i.ident());
-            return mk_lit!("Integer", e_int);
+            return mk_lit!("Integer", suf, e_int);
         }
 
-        token::Literal(token::Float(fident)) => {
+        token::Literal(token::Float(fident), suf) => {
             let e_fident = mk_name(cx, sp, fident.ident());
-            return mk_lit!("Float", e_fident);
+            return mk_lit!("Float", suf, e_fident);
         }
 
-        token::Literal(token::Str_(ident)) => {
-            return mk_lit!("Str_", mk_name(cx, sp, ident.ident()))
+        token::Literal(token::Str_(ident), suf) => {
+            return mk_lit!("Str_", suf, mk_name(cx, sp, ident.ident()))
         }
 
-        token::Literal(token::StrRaw(ident, n)) => {
-            return mk_lit!("StrRaw", mk_name(cx, sp, ident.ident()), cx.expr_uint(sp, n))
+        token::Literal(token::StrRaw(ident, n), suf) => {
+            return mk_lit!("StrRaw", suf, mk_name(cx, sp, ident.ident()), cx.expr_uint(sp, n))
         }
 
         token::Ident(ident, style) => {

branches/snap-stage3/src/libsyntax/parse/lexer/mod.rs

@@ -369,6 +369,25 @@ impl<'a> StringReader<'a> {
         self.nextnextch() == Some(c)
     }
 
+    /// Eats <XID_start><XID_continue>*, if possible.
+    fn scan_optional_raw_name(&mut self) -> Option<ast::Name> {
+        if !ident_start(self.curr) {
+            return None
+        }
+        let start = self.last_pos;
+        while ident_continue(self.curr) {
+            self.bump();
+        }
+
+        self.with_str_from(start, |string| {
+            if string == "_" {
+                None
+            } else {
+                Some(token::intern(string))
+            }
+        })
+    }
+
     /// PRECONDITION: self.curr is not whitespace
     /// Eats any kind of comment.
     fn scan_comment(&mut self) -> Option<TokenAndSpan> {
@@ -638,7 +657,7 @@ impl<'a> StringReader<'a> {
     }
 
     /// Lex a LIT_INTEGER or a LIT_FLOAT
-    fn scan_number(&mut self, c: char) -> token::Token {
+    fn scan_number(&mut self, c: char) -> token::Lit {
         let mut num_digits;
         let mut base = 10;
         let start_bpos = self.last_pos;
@@ -655,17 +674,17 @@ impl<'a> StringReader<'a> {
                 }
                 'u' | 'i' => {
                     self.scan_int_suffix();
-                    return token::Literal(token::Integer(self.name_from(start_bpos)));
+                    return token::Integer(self.name_from(start_bpos));
                 },
                 'f' => {
                     let last_pos = self.last_pos;
                     self.scan_float_suffix();
                     self.check_float_base(start_bpos, last_pos, base);
-                    return token::Literal(token::Float(self.name_from(start_bpos)));
+                    return token::Float(self.name_from(start_bpos));
                 }
                 _ => {
                     // just a 0
-                    return token::Literal(token::Integer(self.name_from(start_bpos)));
+                    return token::Integer(self.name_from(start_bpos));
                 }
             }
         } else if c.is_digit_radix(10) {
@@ -678,7 +697,7 @@ impl<'a> StringReader<'a> {
             self.err_span_(start_bpos, self.last_pos, "no valid digits found for number");
             // eat any suffix
             self.scan_int_suffix();
-            return token::Literal(token::Integer(token::intern("0")));
+            return token::Integer(token::intern("0"));
         }
 
         // might be a float, but don't be greedy if this is actually an
@@ -696,25 +715,25 @@ impl<'a> StringReader<'a> {
             }
             let last_pos = self.last_pos;
             self.check_float_base(start_bpos, last_pos, base);
-            return token::Literal(token::Float(self.name_from(start_bpos)));
+            return token::Float(self.name_from(start_bpos));
         } else if self.curr_is('f') {
             // or it might be an integer literal suffixed as a float
             self.scan_float_suffix();
             let last_pos = self.last_pos;
             self.check_float_base(start_bpos, last_pos, base);
-            return token::Literal(token::Float(self.name_from(start_bpos)));
+            return token::Float(self.name_from(start_bpos));
         } else {
             // it might be a float if it has an exponent
             if self.curr_is('e') || self.curr_is('E') {
                 self.scan_float_exponent();
                 self.scan_float_suffix();
                 let last_pos = self.last_pos;
                 self.check_float_base(start_bpos, last_pos, base);
-                return token::Literal(token::Float(self.name_from(start_bpos)));
+                return token::Float(self.name_from(start_bpos));
             }
             // but we certainly have an integer!
             self.scan_int_suffix();
-            return token::Literal(token::Integer(self.name_from(start_bpos)));
+            return token::Integer(self.name_from(start_bpos));
         }
     }
 
@@ -967,7 +986,9 @@ impl<'a> StringReader<'a> {
         }
 
         if is_dec_digit(c) {
-            return self.scan_number(c.unwrap());
+            let num = self.scan_number(c.unwrap());
+            let suffix = self.scan_optional_raw_name();
+            return token::Literal(num, suffix)
         }
 
         if self.read_embedded_ident {
@@ -1126,17 +1147,19 @@ impl<'a> StringReader<'a> {
             }
             let id = if valid { self.name_from(start) } else { token::intern("0") };
             self.bump(); // advance curr past token
-            return token::Literal(token::Char(id));
+            let suffix = self.scan_optional_raw_name();
+            return token::Literal(token::Char(id), suffix);
           }
           'b' => {
             self.bump();
-            return match self.curr {
+            let lit = match self.curr {
                 Some('\'') => self.scan_byte(),
                 Some('"') => self.scan_byte_string(),
                 Some('r') => self.scan_raw_byte_string(),
                 _ => unreachable!()  // Should have been a token::Ident above.
             };
-
+            let suffix = self.scan_optional_raw_name();
+            return token::Literal(lit, suffix);
           }
           '"' => {
             let start_bpos = self.last_pos;
@@ -1157,7 +1180,8 @@ impl<'a> StringReader<'a> {
             let id = if valid { self.name_from(start_bpos + BytePos(1)) }
                      else { token::intern("??") };
             self.bump();
-            return token::Literal(token::Str_(id));
+            let suffix = self.scan_optional_raw_name();
+            return token::Literal(token::Str_(id), suffix);
           }
           'r' => {
             let start_bpos = self.last_pos;
@@ -1224,7 +1248,8 @@ impl<'a> StringReader<'a> {
             } else {
                 token::intern("??")
             };
-            return token::Literal(token::StrRaw(id, hash_count));
+            let suffix = self.scan_optional_raw_name();
+            return token::Literal(token::StrRaw(id, hash_count), suffix);
           }
           '-' => {
             if self.nextch_is('>') {
@@ -1293,7 +1318,7 @@ impl<'a> StringReader<'a> {
      || (self.curr_is('#') && self.nextch_is('!') && !self.nextnextch_is('['))
     }
 
-    fn scan_byte(&mut self) -> token::Token {
+    fn scan_byte(&mut self) -> token::Lit {
         self.bump();
         let start = self.last_pos;
 
@@ -1314,10 +1339,10 @@ impl<'a> StringReader<'a> {
 
         let id = if valid { self.name_from(start) } else { token::intern("??") };
         self.bump(); // advance curr past token
-        return token::Literal(token::Byte(id));
+        return token::Byte(id);
     }
 
-    fn scan_byte_string(&mut self) -> token::Token {
+    fn scan_byte_string(&mut self) -> token::Lit {
         self.bump();
         let start = self.last_pos;
         let mut valid = true;
@@ -1336,10 +1361,10 @@ impl<'a> StringReader<'a> {
         }
         let id = if valid { self.name_from(start) } else { token::intern("??") };
         self.bump();
-        return token::Literal(token::Binary(id));
+        return token::Binary(id);
     }
 
-    fn scan_raw_byte_string(&mut self) -> token::Token {
+    fn scan_raw_byte_string(&mut self) -> token::Lit {
         let start_bpos = self.last_pos;
         self.bump();
         let mut hash_count = 0u;
@@ -1387,9 +1412,9 @@ impl<'a> StringReader<'a> {
             self.bump();
         }
         self.bump();
-        return token::Literal(token::BinaryRaw(self.name_from_to(content_start_bpos,
-                                                                 content_end_bpos),
-                                               hash_count));
+        return token::BinaryRaw(self.name_from_to(content_start_bpos,
+                                                  content_end_bpos),
+                                hash_count);
     }
 }
 
@@ -1536,17 +1561,17 @@ mod test {
 
     #[test] fn character_a() {
         assert_eq!(setup(&mk_sh(), "'a'".to_string()).next_token().tok,
-                   token::Literal(token::Char(token::intern("a"))));
+                   token::Literal(token::Char(token::intern("a")), None));
     }
 
     #[test] fn character_space() {
         assert_eq!(setup(&mk_sh(), "' '".to_string()).next_token().tok,
-                   token::Literal(token::Char(token::intern(" "))));
+                   token::Literal(token::Char(token::intern(" ")), None));
     }
 
     #[test] fn character_escaped() {
         assert_eq!(setup(&mk_sh(), "'\\n'".to_string()).next_token().tok,
-                   token::Literal(token::Char(token::intern("\\n"))));
+                   token::Literal(token::Char(token::intern("\\n")), None));
     }
 
     #[test] fn lifetime_name() {
@@ -1558,7 +1583,38 @@ mod test {
         assert_eq!(setup(&mk_sh(),
                          "r###\"\"#a\\b\x00c\"\"###".to_string()).next_token()
                                                                  .tok,
-                   token::Literal(token::StrRaw(token::intern("\"#a\\b\x00c\""), 3)));
+                   token::Literal(token::StrRaw(token::intern("\"#a\\b\x00c\""), 3), None));
+    }
+
+    #[test] fn literal_suffixes() {
+        macro_rules! test {
+            ($input: expr, $tok_type: ident, $tok_contents: expr) => {{
+                assert_eq!(setup(&mk_sh(), format!("{}suffix", $input)).next_token().tok,
+                           token::Literal(token::$tok_type(token::intern($tok_contents)),
+                                          Some(token::intern("suffix"))));
+                // with a whitespace separator:
+                assert_eq!(setup(&mk_sh(), format!("{} suffix", $input)).next_token().tok,
+                           token::Literal(token::$tok_type(token::intern($tok_contents)),
+                                          None));
+            }}
+        }
+
+        test!("'a'", Char, "a");
+        test!("b'a'", Byte, "a");
+        test!("\"a\"", Str_, "a");
+        test!("b\"a\"", Binary, "a");
+        test!("1234", Integer, "1234");
+        test!("0b101", Integer, "0b101");
+        test!("0xABC", Integer, "0xABC");
+        test!("1.0", Float, "1.0");
+        test!("1.0e10", Float, "1.0e10");
+
+        assert_eq!(setup(&mk_sh(), "r###\"raw\"###suffix".to_string()).next_token().tok,
+                   token::Literal(token::StrRaw(token::intern("raw"), 3),
+                                  Some(token::intern("suffix"))));
+        assert_eq!(setup(&mk_sh(), "br###\"raw\"###suffix".to_string()).next_token().tok,
+                   token::Literal(token::BinaryRaw(token::intern("raw"), 3),
+                                  Some(token::intern("suffix"))));
     }
 
     #[test] fn line_doc_comments() {
@@ -1574,7 +1630,7 @@ mod test {
             token::Comment => { },
             _ => panic!("expected a comment!")
         }
-        assert_eq!(lexer.next_token().tok, token::Literal(token::Char(token::intern("a"))));
+        assert_eq!(lexer.next_token().tok, token::Literal(token::Char(token::intern("a")), None));
     }
 
 }