---

yaml
---
r: 123705
b: refs/heads/try
c: 9f5e21d
h: refs/heads/master
i:
  123703: 1b1cdee
v: v3

Author: emberian

[refs]

@@ -2,7 +2,7 @@
 refs/heads/master: da4e4e4e0a7778a85748aa4a303b13f603e96b4b
 refs/heads/snap-stage1: e33de59e47c5076a89eadeb38f4934f58a3618a6
 refs/heads/snap-stage3: 8ddd286ea4ba4384a0dc9eae393ed515460a986e
-refs/heads/try: bf04a7ccb1c1d23478885ea4f67fad374ffe0a72
+refs/heads/try: 9f5e21da4ef95e5d2914a76b09848ebc2504c53d
 refs/tags/release-0.1: 1f5c5126e96c79d22cb7862f75304136e204f105
 refs/heads/ndm: f3868061cd7988080c30d6d5bf352a5a5fe2460b
 refs/heads/try2: 147ecfdd8221e4a4d4e090486829a06da1e0ca3c

branches/try/src/libsyntax/ext/base.rs

@@ -579,9 +579,9 @@ pub fn get_single_str_from_tts(cx: &ExtCtxt,
         cx.span_err(sp, format!("{} takes 1 argument.", name).as_slice());
     } else {
         match tts[0] {
-            ast::TTTok(_, token::LIT_STR(ident))
-            | ast::TTTok(_, token::LIT_STR_RAW(ident, _)) => {
-                return Some(token::get_ident(ident).get().to_string())
+            ast::TTTok(_, token::LIT_STR(ident)) => return Some(parse::str_lit(ident.as_str())),
+            ast::TTTok(_, token::LIT_STR_RAW(ident, _)) => {
+                return Some(parse::raw_str_lit(ident.as_str()))
             }
             _ => {
                 cx.span_err(sp,

branches/try/src/libsyntax/ext/quote.rs

@@ -401,13 +401,13 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> Gc<ast::Expr> {
         }
 
         LIT_BYTE(i) => {
-            let e_byte = cx.expr_lit(sp, ast::LitByte(i));
+            let e_byte = mk_ident(cx, sp, i);
 
             return cx.expr_call(sp, mk_token_path(cx, sp, "LIT_BYTE"), vec!(e_byte));
         }
 
         LIT_CHAR(i) => {
-            let e_char = cx.expr_lit(sp, ast::LitChar(i));
+            let e_char = mk_ident(cx, sp, i);
 
             return cx.expr_call(sp, mk_token_path(cx, sp, "LIT_CHAR"), vec!(e_char));
         }

branches/try/src/libsyntax/parse/lexer/mod.rs

@@ -685,7 +685,7 @@ impl<'a> StringReader<'a> {
     }
 
 
-    fn scan_numeric_escape(&mut self, n_hex_digits: uint, delim: char) -> char {
+    fn scan_numeric_escape(&mut self, n_hex_digits: uint, delim: char) -> bool {
         let mut accum_int = 0u32;
         let start_bpos = self.last_pos;
         for _ in range(0, n_hex_digits) {
@@ -709,20 +709,22 @@ impl<'a> StringReader<'a> {
         }
 
         match char::from_u32(accum_int) {
-            Some(x) => x,
+            Some(_) => true,
             None => {
                 let last_bpos = self.last_pos;
                 self.err_span_(start_bpos, last_bpos, "illegal numeric character escape");
-                '?'
+                false
             }
         }
     }
 
     /// Scan for a single (possibly escaped) byte or char
     /// in a byte, (non-raw) byte string, char, or (non-raw) string literal.
     /// `start` is the position of `first_source_char`, which is already consumed.
+    ///
+    /// Returns true if there was a valid char/byte, false otherwise.
     fn scan_char_or_byte(&mut self, start: BytePos, first_source_char: char,
-                         ascii_only: bool, delim: char) -> Option<char> {
+                         ascii_only: bool, delim: char) -> bool {
         match first_source_char {
             '\\' => {
                 // '\X' for some X must be a character constant:
@@ -732,24 +734,18 @@ impl<'a> StringReader<'a> {
                 match escaped {
                     None => {},  // EOF here is an error that will be checked later.
                     Some(e) => {
-                        return Some(match e {
-                            'n' => '\n',
-                            'r' => '\r',
-                            't' => '\t',
-                            '\\' => '\\',
-                            '\'' => '\'',
-                            '"' => '"',
-                            '0' => '\x00',
+                        return match e {
+                            'n' | 'r' | 't' | '\\' | '\'' | '"' | '0' => true,
                             'x' => self.scan_numeric_escape(2u, delim),
                             'u' if !ascii_only => self.scan_numeric_escape(4u, delim),
                             'U' if !ascii_only => self.scan_numeric_escape(8u, delim),
                             '\n' if delim == '"' => {
                                 self.consume_whitespace();
-                                return None
+                                true
                             },
                             '\r' if delim == '"' && self.curr_is('\n') => {
                                 self.consume_whitespace();
-                                return None
+                                true
                             }
                             c => {
                                 let last_pos = self.last_pos;
@@ -758,9 +754,9 @@ impl<'a> StringReader<'a> {
                                     if ascii_only { "unknown byte escape" }
                                     else { "unknown character escape" },
                                     c);
-                                c
+                                false
                             }
-                        })
+                        }
                     }
                 }
             }
@@ -771,14 +767,16 @@ impl<'a> StringReader<'a> {
                     if ascii_only { "byte constant must be escaped" }
                     else { "character constant must be escaped" },
                     first_source_char);
+                return false;
             }
             '\r' => {
                 if self.curr_is('\n') {
                     self.bump();
-                    return Some('\n');
+                    return true;
                 } else {
                     self.err_span_(start, self.last_pos,
                                    "bare CR not allowed in string, use \\r instead");
+                    return false;
                 }
             }
             _ => if ascii_only && first_source_char > '\x7F' {
@@ -787,9 +785,10 @@ impl<'a> StringReader<'a> {
                     start, last_pos,
                     "byte constant must be ASCII. \
                      Use a \\xHH escape for a non-ASCII byte", first_source_char);
+                return false;
             }
         }
-        Some(first_source_char)
+        true
     }
 
     fn binop(&mut self, op: token::BinOp) -> token::Token {
@@ -924,7 +923,7 @@ impl<'a> StringReader<'a> {
             let start = self.last_pos;
 
             // the eof will be picked up by the final `'` check below
-            let mut c2 = self.curr.unwrap_or('\x00');
+            let c2 = self.curr.unwrap_or('\x00');
             self.bump();
 
             // If the character is an ident start not followed by another single
@@ -967,7 +966,7 @@ impl<'a> StringReader<'a> {
             }
 
             // Otherwise it is a character constant:
-            c2 = self.scan_char_or_byte(start, c2, /* ascii_only = */ false, '\'').unwrap();
+            let valid = self.scan_char_or_byte(start, c2, /* ascii_only = */ false, '\'');
             if !self.curr_is('\'') {
                 let last_bpos = self.last_pos;
                 self.fatal_span_verbose(
@@ -977,8 +976,9 @@ impl<'a> StringReader<'a> {
                                    start - BytePos(1), last_bpos,
                                    "unterminated character constant".to_string());
             }
+            let id = if valid { self.ident_from(start) } else { str_to_ident("0") };
             self.bump(); // advance curr past token
-            return token::LIT_CHAR(c2);
+            return token::LIT_CHAR(id);
           }
           'b' => {
             self.bump();
@@ -991,8 +991,8 @@ impl<'a> StringReader<'a> {
 
           }
           '"' => {
-            let mut accum_str = String::new();
             let start_bpos = self.last_pos;
+            let mut valid = true;
             self.bump();
             while !self.curr_is('"') {
                 if self.is_eof() {
@@ -1003,11 +1003,13 @@ impl<'a> StringReader<'a> {
                 let ch_start = self.last_pos;
                 let ch = self.curr.unwrap();
                 self.bump();
-                self.scan_char_or_byte(ch_start, ch, /* ascii_only = */ false, '"')
-                    .map(|ch| accum_str.push_char(ch));
+                valid &= self.scan_char_or_byte(ch_start, ch, /* ascii_only = */ false, '"');
             }
+            // adjust for the ACSII " at the start of the literal
+            let id = if valid { self.ident_from(start_bpos + BytePos(1)) }
+                     else { str_to_ident("??") };
             self.bump();
-            return token::LIT_STR(str_to_ident(accum_str.as_slice()));
+            return token::LIT_STR(id);
           }
           'r' => {
             let start_bpos = self.last_pos;
@@ -1032,7 +1034,7 @@ impl<'a> StringReader<'a> {
             self.bump();
             let content_start_bpos = self.last_pos;
             let mut content_end_bpos;
-            let mut has_cr = false;
+            let mut valid = true;
             'outer: loop {
                 if self.is_eof() {
                     let last_bpos = self.last_pos;
@@ -1055,23 +1057,26 @@ impl<'a> StringReader<'a> {
                             }
                         }
                         break;
-                    }
+                    },
                     '\r' => {
-                        has_cr = true;
+                        if !self.nextch_is('\n') {
+                            let last_bpos = self.last_pos;
+                            self.err_span_(start_bpos, last_bpos, "bare CR not allowed in raw \
+                                           string, use \\r instead");
+                            valid = false;
+                        }
                     }
                     _ => ()
                 }
                 self.bump();
             }
             self.bump();
-            let str_content = self.with_str_from_to(content_start_bpos, content_end_bpos, |string| {
-                let string = if has_cr {
-                    self.translate_crlf(content_start_bpos, string,
-                                        "bare CR not allowed in raw string")
-                } else { string.into_maybe_owned() };
-                str_to_ident(string.as_slice())
-            });
-            return token::LIT_STR_RAW(str_content, hash_count);
+            let id = if valid {
+                self.ident_from_to(content_start_bpos, content_end_bpos)
+            } else {
+                str_to_ident("??")
+            };
+            return token::LIT_STR_RAW(id, hash_count);
           }
           '-' => {
             if self.nextch_is('>') {
@@ -1145,10 +1150,10 @@ impl<'a> StringReader<'a> {
         let start = self.last_pos;
 
         // the eof will be picked up by the final `'` check below
-        let mut c2 = self.curr.unwrap_or('\x00');
+        let c2 = self.curr.unwrap_or('\x00');
         self.bump();
 
-        c2 = self.scan_char_or_byte(start, c2, /* ascii_only = */ true, '\'').unwrap();
+        let valid = self.scan_char_or_byte(start, c2, /* ascii_only = */ true, '\'');
         if !self.curr_is('\'') {
             // Byte offsetting here is okay because the
             // character before position `start` are an
@@ -1158,14 +1163,17 @@ impl<'a> StringReader<'a> {
                 start - BytePos(2), last_pos,
                 "unterminated byte constant".to_string());
         }
+
+        let id = if valid { self.ident_from(start) } else { str_to_ident("??") };
         self.bump(); // advance curr past token
-        return token::LIT_BYTE(c2 as u8);
+        return token::LIT_BYTE(id);
     }
 
     fn scan_byte_string(&mut self) -> token::Token {
         self.bump();
         let start = self.last_pos;
-        let mut value = Vec::new();
+        let mut valid = true;
+
         while !self.curr_is('"') {
             if self.is_eof() {
                 let last_pos = self.last_pos;
@@ -1176,11 +1184,11 @@ impl<'a> StringReader<'a> {
             let ch_start = self.last_pos;
             let ch = self.curr.unwrap();
             self.bump();
-            self.scan_char_or_byte(ch_start, ch, /* ascii_only = */ true, '"')
-                .map(|ch| value.push(ch as u8));
+            valid &= self.scan_char_or_byte(ch_start, ch, /* ascii_only = */ true, '"');
         }
+        let id = if valid { self.ident_from(start) } else { str_to_ident("??") };
         self.bump();
-        return token::LIT_BINARY(Rc::new(value));
+        return token::LIT_BINARY(id);
     }
 
     fn scan_raw_byte_string(&mut self) -> token::Token {
@@ -1231,10 +1239,8 @@ impl<'a> StringReader<'a> {
             self.bump();
         }
         self.bump();
-        let bytes = self.with_str_from_to(content_start_bpos,
-                                           content_end_bpos,
-                                           |s| s.as_bytes().to_owned());
-        return token::LIT_BINARY_RAW(Rc::new(bytes), hash_count);
+        return token::LIT_BINARY_RAW(self.ident_from_to(content_start_bpos, content_end_bpos),
+                                     hash_count);
     }
 }