auto merge of #6083 : jbclements/rust/parser-cleanup, r=jbclements

r? @pcwalton

A month's worth of parser cleanup here. Much of this is new comments and renaming. A number of these commits also remove unneeded code.  Probably the biggest refactor here is splitting "parse_item_or_view_item" into two functions; it turns out that the only overlap between items in foreign modules and items in regular modules was macros, so this refactor should make things substantially easier for future maintenance.

Author: bors

src/librustc/driver/driver.rs

@@ -149,7 +149,7 @@ pub fn parse_input(sess: Session, cfg: ast::crate_cfg, input: &input)
     -> @ast::crate {
     match *input {
       file_input(ref file) => {
-        parse::parse_crate_from_file_using_tts(&(*file), cfg, sess.parse_sess)
+        parse::parse_crate_from_file(&(*file), cfg, sess.parse_sess)
       }
       str_input(ref src) => {
         // FIXME (#2319): Don't really want to box the source string

src/librustdoc/attr_parser.rs

@@ -79,7 +79,7 @@ mod test {
 
         let parse_sess = syntax::parse::new_parse_sess(None);
         let parser = parse::new_parser_from_source_str(
-            parse_sess, ~[], ~"-", codemap::FssNone, @source);
+            parse_sess, ~[], ~"-", @source);
 
         parser.parse_outer_attributes()
     }

src/librustpkg/tests.rs

@@ -66,7 +66,7 @@ fn test_make_dir_rwx() {
 fn test_install_valid() {
     let ctxt = fake_ctxt();
     let temp_pkg_id = fake_pkg();
-    let temp_workspace() = mk_temp_workspace();
+    let temp_workspace = mk_temp_workspace();
     // should have test, bench, lib, and main
     ctxt.install(&temp_workspace, temp_pkg_id);
     // Check that all files exist

src/libsyntax/ast_util.rs

@@ -355,6 +355,10 @@ pub fn operator_prec(op: ast::binop) -> uint {
   }
 }
 
+/// Precedence of the `as` operator, which is a binary operator
+/// not appearing in the prior table.
+pub static as_prec: uint = 11u;
+
 pub fn dtor_ty() -> @ast::Ty {
     @ast::Ty {id: 0, node: ty_nil, span: dummy_sp()}
 }
@@ -756,7 +760,6 @@ mod test {
         assert_eq!(refold_test_sc(3,&t),test_sc);
     }
 
-
     // extend a syntax context with a sequence of marks given
     // in a vector. v[0] will be the outermost mark.
     fn unfold_marks(mrks:~[Mrk],tail:SyntaxContext,table: &mut SCTable) -> SyntaxContext {

src/libsyntax/ext/base.rs

@@ -461,9 +461,7 @@ impl <K: Eq + Hash + IterBytes ,V: Copy> MapChain<K,V>{
 
     // ugh: can't get this to compile with mut because of the
     // lack of flow sensitivity.
-    #[cfg(stage1)]
-    #[cfg(stage2)]
-    #[cfg(stage3)]
+    #[cfg(not(stage0))]
     fn get_map<'a>(&'a self) -> &'a HashMap<K,@V> {
         match *self {
             BaseMapChain (~ref map) => map,

src/libsyntax/opt_vec.rs

@@ -69,9 +69,7 @@ impl<T> OptVec<T> {
         }
     }
 
-    #[cfg(stage1)]
-    #[cfg(stage2)]
-    #[cfg(stage3)]
+    #[cfg(not(stage0))]
     fn get<'a>(&'a self, i: uint) -> &'a T {
         match *self {
             Empty => fail!(fmt!("Invalid index %u", i)),

src/libsyntax/parse/attr.rs

@@ -62,12 +62,14 @@ impl parser_attr for Parser {
         return attrs;
     }
 
+    // matches attribute = # attribute_naked
     fn parse_attribute(&self, style: ast::attr_style) -> ast::attribute {
         let lo = self.span.lo;
         self.expect(&token::POUND);
         return self.parse_attribute_naked(style, lo);
     }
 
+    // matches attribute_naked = [ meta_item ]
     fn parse_attribute_naked(&self, style: ast::attr_style, lo: BytePos) ->
         ast::attribute {
         self.expect(&token::LBRACKET);
@@ -86,6 +88,7 @@ impl parser_attr for Parser {
     // is an inner attribute of the containing item or an outer attribute of
     // the first contained item until we see the semi).
 
+    // matches inner_attrs* outer_attr?
     // you can make the 'next' field an Option, but the result is going to be
     // more useful as a vector.
     fn parse_inner_attrs_and_next(&self) ->
@@ -134,6 +137,9 @@ impl parser_attr for Parser {
         (inner_attrs, next_outer_attrs)
     }
 
+    // matches meta_item = IDENT
+    // | IDENT = lit
+    // | IDENT meta_seq
     fn parse_meta_item(&self) -> @ast::meta_item {
         let lo = self.span.lo;
         let name = self.id_to_str(self.parse_ident());
@@ -156,6 +162,7 @@ impl parser_attr for Parser {
         }
     }
 
+    // matches meta_seq = ( COMMASEP(meta_item) )
     fn parse_meta_seq(&self) -> ~[@ast::meta_item] {
         copy self.parse_seq(
             &token::LPAREN,

src/libsyntax/parse/classify.rs

@@ -15,6 +15,13 @@
 use ast;
 use codemap;
 
+// does this expression require a semicolon to be treated
+// as a statement? The negation of this: 'can this expression
+// be used as a statement without a semicolon' -- is used
+// as an early-bail-out in the parser so that, for instance,
+// 'if true {...} else {...}
+//  |x| 5 '
+// isn't parsed as (if true {...} else {...} | x) | 5
 pub fn expr_requires_semi_to_be_stmt(e: @ast::expr) -> bool {
     match e.node {
       ast::expr_if(*)
@@ -40,6 +47,9 @@ pub fn expr_is_simple_block(e: @ast::expr) -> bool {
     }
 }
 
+// this statement requires a semicolon after it.
+// note that in one case (stmt_semi), we've already
+// seen the semicolon, and thus don't need another.
 pub fn stmt_ends_with_semi(stmt: &ast::stmt) -> bool {
     return match stmt.node {
         ast::stmt_decl(d, _) => {

src/libsyntax/parse/comments.rs

@@ -309,6 +309,8 @@ pub struct lit {
     pos: BytePos
 }
 
+// it appears this function is called only from pprust... that's
+// probably not a good thing.
 pub fn gather_comments_and_literals(span_diagnostic:
                                     @diagnostic::span_handler,
                                     path: ~str,

src/libsyntax/parse/lexer.rs

@@ -225,20 +225,12 @@ pub fn is_whitespace(c: char) -> bool {
     return c == ' ' || c == '\t' || c == '\r' || c == '\n';
 }
 
-fn may_begin_ident(c: char) -> bool { return is_alpha(c) || c == '_'; }
-
 fn in_range(c: char, lo: char, hi: char) -> bool {
     return lo <= c && c <= hi
 }
 
-fn is_alpha(c: char) -> bool {
-    return in_range(c, 'a', 'z') || in_range(c, 'A', 'Z');
-}
-
 fn is_dec_digit(c: char) -> bool { return in_range(c, '0', '9'); }
 
-fn is_alnum(c: char) -> bool { return is_alpha(c) || is_dec_digit(c); }
-
 fn is_hex_digit(c: char) -> bool {
     return in_range(c, '0', '9') || in_range(c, 'a', 'f') ||
             in_range(c, 'A', 'F');
@@ -294,6 +286,8 @@ fn consume_any_line_comment(rdr: @mut StringReader)
         }
     } else if rdr.curr == '#' {
         if nextch(rdr) == '!' {
+            // I guess this is the only way to figure out if
+            // we're at the beginning of the file...
             let cmap = @CodeMap::new();
             (*cmap).files.push(rdr.filemap);
             let loc = cmap.lookup_char_pos_adj(rdr.last_pos);
@@ -444,8 +438,7 @@ fn scan_number(c: char, rdr: @mut StringReader) -> token::Token {
         }
     }
     let mut is_float = false;
-    if rdr.curr == '.' && !(is_alpha(nextch(rdr)) || nextch(rdr) == '_' ||
-                            nextch(rdr) == '.') {
+    if rdr.curr == '.' && !(ident_start(nextch(rdr)) || nextch(rdr) == '.') {
         is_float = true;
         bump(rdr);
         let dec_part = scan_digits(rdr, 10u);