Make it possible to make tt_reader-based parsers.

Author: paulstansifer

src/libsyntax/parse.rs

@@ -18,7 +18,8 @@ import attr::parser_attr;
 import common::parser_common;
 import ast::node_id;
 import util::interner;
-import lexer::{string_reader_as_reader, reader, string_reader};
+import lexer::{string_reader_as_reader, tt_reader_as_reader,
+               reader, string_reader, tt_reader};
 
 type parse_sess = @{
     cm: codemap::codemap,
@@ -178,3 +179,10 @@ fn new_parser_from_file(sess: parse_sess, cfg: ast::crate_cfg, +path: str,
     let (p, _) = new_parser_etc_from_file(sess, cfg, path, ftype);
     ret p;
 }
+
+fn new_parser_from_tt(sess: parse_sess, cfg: ast::crate_cfg,
+                      itr: @interner::interner<@str>, tt: ast::token_tree)
+    -> parser {
+    let trdr = lexer::new_tt_reader(sess.span_diagnostic, itr, tt);
+    ret parser(sess, cfg, trdr as reader, parser::SOURCE_FILE)
+}

src/libsyntax/parse/lexer.rs

@@ -4,8 +4,9 @@ import diagnostic;
 import ast::{tt_delim,tt_flat};
 
 export reader, string_reader, new_string_reader, is_whitespace;
+export tt_reader, new_tt_reader;
 export nextch, is_eof, bump, get_str_from;
-export string_reader_as_reader;
+export string_reader_as_reader, tt_reader_as_reader;
 
 iface reader {
     fn is_eof() -> bool;
@@ -27,14 +28,27 @@ type tt_frame = @{
 };
 
 type tt_reader = @{
-    mut cur: tt_frame,
-    interner: @interner::interner<@str>,
     span_diagnostic: diagnostic::span_handler,
+    interner: @interner::interner<@str>,
+    mut cur: tt_frame,
     /* cached: */
     mut cur_tok: token::token,
     mut cur_chpos: uint
 };
 
+fn new_tt_reader(span_diagnostic: diagnostic::span_handler,
+                 itr: @interner::interner<@str>, src: ast::token_tree)
+    -> tt_reader {
+    let r = @{span_diagnostic: span_diagnostic, interner: itr,
+              mut cur: @{readme: [src], mut idx: 0u,
+                         up: tt_frame_up(option::none)},
+              mut cur_tok: token::EOF, /* dummy value, never read */
+              mut cur_chpos: 0u /* dummy value, never read */
+             };
+    (r as reader).next_token(); /* get cur_tok and cur_chpos set up */
+    ret r;
+}
+
 type string_reader = @{
     span_diagnostic: diagnostic::span_handler,
     src: @str,
@@ -46,6 +60,21 @@ type string_reader = @{
     interner: @interner::interner<@str>
 };
 
+fn new_string_reader(span_diagnostic: diagnostic::span_handler,
+                     filemap: codemap::filemap,
+                     itr: @interner::interner<@str>) -> string_reader {
+    let r = @{span_diagnostic: span_diagnostic, src: filemap.src,
+              mut col: 0u, mut pos: 0u, mut curr: -1 as char,
+              mut chpos: filemap.start_pos.ch,
+              filemap: filemap, interner: itr};
+    if r.pos < (*filemap.src).len() {
+        let next = str::char_range_at(*r.src, r.pos);
+        r.pos = next.next;
+        r.curr = next.ch;
+    }
+    ret r;
+}
+
 impl string_reader_as_reader of reader for string_reader {
     fn is_eof() -> bool { is_eof(self) }
     fn next_token() -> {tok: token::token, chpos: uint} {
@@ -70,7 +99,6 @@ impl tt_reader_as_reader of reader for tt_reader {
     fn is_eof() -> bool { self.cur_tok == token::EOF }
     fn next_token() -> {tok: token::token, chpos: uint} {
         let ret_val = { tok: self.cur_tok, chpos: self.cur_chpos };
-        self.cur.idx += 1u;
         if self.cur.idx >= vec::len(self.cur.readme) {
             /* done with this set; pop */
             alt self.cur.up {
@@ -97,6 +125,7 @@ impl tt_reader_as_reader of reader for tt_reader {
               }
               tt_flat(chpos, tok) {
                 self.cur_chpos = chpos; self.cur_tok = tok;
+                self.cur.idx += 1u;
                 ret ret_val;
               }
           }
@@ -144,21 +173,6 @@ fn nextch(rdr: string_reader) -> char {
     } else { ret -1 as char; }
 }
 
-fn new_string_reader(span_diagnostic: diagnostic::span_handler,
-                     filemap: codemap::filemap,
-                     itr: @interner::interner<@str>) -> string_reader {
-    let r = @{span_diagnostic: span_diagnostic, src: filemap.src,
-              mut col: 0u, mut pos: 0u, mut curr: -1 as char,
-              mut chpos: filemap.start_pos.ch,
-              filemap: filemap, interner: itr};
-    if r.pos < (*filemap.src).len() {
-        let next = str::char_range_at(*r.src, r.pos);
-        r.pos = next.next;
-        r.curr = next.ch;
-    }
-    ret r;
-}
-
 fn dec_digit_val(c: char) -> int { ret (c as int) - ('0' as int); }
 
 fn hex_digit_val(c: char) -> int {