Use a hand-written parser a string formatting for the match_token! macro…

… instead of the parser and quasi-quoting from Rust’s (unstable) libsyntax.

This has significantly worse diagnostics when encountering unexpected syntax
(e.g. no indication of which line has the offending code)
but this removes all usage of unstable compiler internals
that constantly need to be fixed when updating the compiler.

Fixes #216.

Author: SimonSapin

Cargo.toml

@@ -33,7 +33,7 @@ harness = false
 [features]
 unstable = ["tendril/unstable", "string_cache/unstable"]
 heap_size = ["heapsize", "heapsize_plugin"]
-codegen = ["html5ever_macros"]
+codegen = []  # Now unused, remove at the next breaking change
 
 [dependencies]
 log = "0"
@@ -51,7 +51,6 @@ rustc-test = "0.1.3"
 [build-dependencies]
 phf_codegen = "0.7.3"
 rustc-serialize = "0.3.15"
-html5ever_macros = { version = "0.2.6", path = "macros", optional = true }
 
 [profile.dev]
 debug = false

STRUCTURE.md

@@ -14,7 +14,7 @@ The module structure is also documented in the output produced by `cargo doc`, a
 
 `dom_sink/`: Types that html5ever can use to represent the DOM, if you do not provide your own DOM implementation.
 
-`macros/`: Rust syntax extensions used within html5ever.  Users of the library do not need this crate.
+`macros/`: Code used at build-time to expand the `match_token!` "macro" in `src/tree_builder/rules.rs`.
 
 `tests/`: Integration tests. This is a single executable crate that runs html5ever on the various [html5lib-tests](https://github.com/html5lib/html5lib-tests). There are also unit tests throughout the library code. See `README.md` for information on running tests.
 

build.rs

@@ -18,14 +18,16 @@ use std::fs::File;
 use std::io::Write;
 use std::path::Path;
 
+#[path = "macros/match_token.rs"]
+mod match_token;
+
 fn main() {
     let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
 
     let rules_rs = Path::new(&manifest_dir).join("src/tree_builder/rules.rs");
-    expand_match_tokens(
+    match_token::expand_match_tokens(
         &rules_rs,
-        // Keep the expanded file in the source directory, so that `cargo publish` ships it.
-        &rules_rs.with_extension("expanded.rs"));
+        &Path::new(&env::var("OUT_DIR").unwrap()).join("rules.rs"));
 
     named_entities_to_phf(
         &Path::new(&manifest_dir).join("data/entities.json"),
@@ -34,67 +36,6 @@ fn main() {
     println!("cargo:rerun-if-changed={}", rules_rs.display());
 }
 
-#[cfg(feature = "codegen")]
-fn expand_match_tokens(from: &Path, to: &Path) {
-    extern crate html5ever_macros;
-
-    html5ever_macros::pre_expand(from, to);
-}
-
-#[cfg(not(feature = "codegen"))]
-fn expand_match_tokens(from: &Path, to: &Path) {
-    use std::io::stderr;
-    use std::process::exit;
-
-    if let Err(error) = check_hash(from, to) {
-        writeln!(
-            stderr(),
-            r"
-{} is missing or not up to date with {}:
-{}
-
-Run `cargo build --features codegen` to update it.
-
-If you’re using html5ever as a dependency, this is a bad release.
-Please file an issue at https://github.com/servo/html5ever/issues/new
-with the output of `cargo pkgid html5ever`.
-",
-            to.file_name().unwrap().to_string_lossy(),
-            from.file_name().unwrap().to_string_lossy(),
-            error
-        ).unwrap();
-        exit(1);
-    }
-}
-
-#[cfg(not(feature = "codegen"))]
-fn check_hash(from: &Path, to: &Path) -> Result<(), String> {
-    use std::hash::{Hash, Hasher, SipHasher};
-    use std::io::Read;
-
-    // Unwrap here as the source file is expected to exist.
-    let mut file_from = File::open(from).unwrap();
-    let mut source = String::new();
-    let mut hasher = SipHasher::new();
-    file_from.read_to_string(&mut source).unwrap();
-    source.hash(&mut hasher);
-    let source_hash = hasher.finish();
-
-    // IO errors from here indicate we need to regenerate the expanded file.
-    let mut file_to = try!(File::open(to).map_err(|e| e.to_string()));
-    let mut expanded = String::new();
-    try!(file_to.read_to_string(&mut expanded).map_err(|e| e.to_string()));
-    let prefix = "// source SipHash: ";
-    let line = try!(expanded.lines().find(|line| line.starts_with(prefix))
-                    .ok_or("source hash not found".to_string()));
-    let expected_hash = try!(line[prefix.len()..].parse::<u64>().map_err(|e| e.to_string()));
-    if source_hash == expected_hash {
-        Ok(())
-    } else {
-        Err("different hash".to_string())
-    }
-}
-
 fn named_entities_to_phf(from: &Path, to: &Path) {
     // A struct matching the entries in entities.json.
     #[derive(RustcDecodable)]