Code to save/load the work-products map from disk

nikomatsakis · nikomatsakis · commit cca480425195 · 2016-07-28T12:05:04.000-04:00
Work products are deleted if any of their inputs are dirty.
diff --git a/src/librustc_driver/driver.rs b/src/librustc_driver/driver.rs
@@ -88,7 +88,7 @@ pub fn compile_input(sess: &Session,
     // We need nested scopes here, because the intermediate results can keep
     // large chunks of memory alive and we want to free them as soon as
     // possible to keep the peak memory usage low
-    let (outputs, trans) = {
+    let (outputs, trans, id) = {
         let krate = match phase_1_parse_input(sess, cfg, input) {
             Ok(krate) => krate,
             Err(mut parse_error) => {
@@ -212,11 +212,11 @@ pub fn compile_input(sess: &Session,
             // Discard interned strings as they are no longer required.
             token::clear_ident_interner();
 
-            Ok((outputs, trans))
+            Ok((outputs, trans, id.clone()))
         })??
     };
 
-    let phase5_result = phase_5_run_llvm_passes(sess, &trans, &outputs);
+    let phase5_result = phase_5_run_llvm_passes(sess, &id, &trans, &outputs);
 
     controller_entry_point!(after_llvm,
                             sess,
@@ -1020,6 +1020,7 @@ pub fn phase_4_translate_to_llvm<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
 /// Run LLVM itself, producing a bitcode file, assembly file or object file
 /// as a side effect.
 pub fn phase_5_run_llvm_passes(sess: &Session,
+                               crate_name: &str,
                                trans: &trans::CrateTranslation,
                                outputs: &OutputFilenames) -> CompileResult {
     if sess.opts.cg.no_integrated_as {
@@ -1041,6 +1042,10 @@ pub fn phase_5_run_llvm_passes(sess: &Session,
              || write::run_passes(sess, trans, &sess.opts.output_types, outputs));
     }
 
+    time(sess.time_passes(),
+         "serialize work products",
+         move || rustc_incremental::save_work_products(sess, crate_name));
+
     if sess.err_count() > 0 {
         Err(sess.err_count())
     } else {
diff --git a/src/librustc_incremental/lib.rs b/src/librustc_incremental/lib.rs
@@ -19,6 +19,7 @@
       html_root_url = "https://doc.rust-lang.org/nightly/")]
 #![cfg_attr(not(stage0), deny(warnings))]
 
+#![feature(question_mark)]
 #![feature(rustc_private)]
 #![feature(staged_api)]
 
@@ -40,3 +41,4 @@ pub use assert_dep_graph::assert_dep_graph;
 pub use calculate_svh::SvhCalculate;
 pub use persist::load_dep_graph;
 pub use persist::save_dep_graph;
+pub use persist::save_work_products;
diff --git a/src/librustc_incremental/persist/data.rs b/src/librustc_incremental/persist/data.rs
@@ -10,8 +10,9 @@
 
 //! The data that we will serialize and deserialize.
 
-use rustc::dep_graph::DepNode;
+use rustc::dep_graph::{DepNode, WorkProduct, WorkProductId};
 use rustc::hir::def_id::DefIndex;
+use std::sync::Arc;
 
 use super::directory::DefPathIndex;
 
@@ -55,6 +56,15 @@ pub struct SerializedHash {
     pub hash: u64,
 }
 
+#[derive(Debug, RustcEncodable, RustcDecodable)]
+pub struct SerializedWorkProduct {
+    /// node that produced the work-product
+    pub id: Arc<WorkProductId>,
+
+    /// work-product data itself
+    pub work_product: WorkProduct,
+}
+
 /// Data for use when downstream crates get recompiled.
 #[derive(Debug, RustcEncodable, RustcDecodable)]
 pub struct SerializedMetadataHashes {
diff --git a/src/librustc_incremental/persist/load.rs b/src/librustc_incremental/persist/load.rs
@@ -14,12 +14,13 @@ use rbml::Error;
 use rbml::opaque::Decoder;
 use rustc::dep_graph::DepNode;
 use rustc::hir::def_id::DefId;
+use rustc::session::Session;
 use rustc::ty::TyCtxt;
 use rustc_data_structures::fnv::FnvHashSet;
 use rustc_serialize::Decodable as RustcDecodable;
 use std::io::Read;
-use std::fs::File;
-use std::path::Path;
+use std::fs::{self, File};
+use std::path::{Path};
 
 use super::data::*;
 use super::directory::*;
@@ -38,50 +39,73 @@ type CleanEdges = Vec<(DepNode<DefId>, DepNode<DefId>)>;
 /// actually it doesn't matter all that much.) See `README.md` for
 /// more general overview.
 pub fn load_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>) {
+    if tcx.sess.opts.incremental.is_none() {
+        return;
+    }
+
     let _ignore = tcx.dep_graph.in_ignore();
+    load_dep_graph_if_exists(tcx);
+    dirty_clean::check_dirty_clean_annotations(tcx);
+}
 
-    if let Some(dep_graph) = dep_graph_path(tcx) {
-        // FIXME(#32754) lock file?
-        load_dep_graph_if_exists(tcx, &dep_graph);
-        dirty_clean::check_dirty_clean_annotations(tcx);
+fn load_dep_graph_if_exists<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>) {
+    let dep_graph_path = dep_graph_path(tcx).unwrap();
+    let dep_graph_data = match load_data(tcx.sess, &dep_graph_path) {
+        Some(p) => p,
+        None => return // no file
+    };
+
+    let work_products_path = tcx_work_products_path(tcx).unwrap();
+    let work_products_data = match load_data(tcx.sess, &work_products_path) {
+        Some(p) => p,
+        None => return // no file
+    };
+
+    match decode_dep_graph(tcx, &dep_graph_data, &work_products_data) {
+        Ok(()) => return,
+        Err(err) => bug!("decoding error in dep-graph from `{}` and `{}`: {}",
+                         dep_graph_path.display(),
+                         work_products_path.display(),
+                         err),
     }
 }
 
-pub fn load_dep_graph_if_exists<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, path: &Path) {
+fn load_data(sess: &Session, path: &Path) -> Option<Vec<u8>> {
     if !path.exists() {
-        return;
+        return None;
     }
 
     let mut data = vec![];
     match
         File::open(path)
         .and_then(|mut file| file.read_to_end(&mut data))
     {
-        Ok(_) => { }
+        Ok(_) => {
+            Some(data)
+        }
         Err(err) => {
-            tcx.sess.err(
+            sess.err(
                 &format!("could not load dep-graph from `{}`: {}",
                          path.display(), err));
-            return;
+            None
         }
     }
 
-    match decode_dep_graph(tcx, &data) {
-        Ok(dirty) => dirty,
-        Err(err) => {
-            bug!("decoding error in dep-graph from `{}`: {}", path.display(), err);
-        }
-    }
 }
 
+/// Decode the dep graph and load the edges/nodes that are still clean
+/// into `tcx.dep_graph`. On success, returns a hashset containing all
+/// the paths of work-products from clean nodes (any work-products not
+/// in this set can be deleted).
 pub fn decode_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
-                                  data: &[u8])
+                                  dep_graph_data: &[u8],
+                                  work_products_data: &[u8])
                                   -> Result<(), Error>
 {
     // Deserialize the directory and dep-graph.
-    let mut decoder = Decoder::new(data, 0);
-    let directory = try!(DefIdDirectory::decode(&mut decoder));
-    let serialized_dep_graph = try!(SerializedDepGraph::decode(&mut decoder));
+    let mut dep_graph_decoder = Decoder::new(dep_graph_data, 0);
+    let directory = try!(DefIdDirectory::decode(&mut dep_graph_decoder));
+    let serialized_dep_graph = try!(SerializedDepGraph::decode(&mut dep_graph_decoder));
 
     debug!("decode_dep_graph: directory = {:#?}", directory);
     debug!("decode_dep_graph: serialized_dep_graph = {:#?}", serialized_dep_graph);
@@ -121,12 +145,18 @@ pub fn decode_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
     // Add nodes and edges that are not dirty into our main graph.
     let dep_graph = tcx.dep_graph.clone();
     for (source, target) in clean_edges.into_iter().chain(clean_nodes) {
-        let _task = dep_graph.in_task(target.clone());
-        dep_graph.read(source.clone());
-
         debug!("decode_dep_graph: clean edge: {:?} -> {:?}", source, target);
+
+        let _task = dep_graph.in_task(target);
+        dep_graph.read(source);
     }
 
+    // Add in work-products that are still clean, and delete those that are
+    // dirty.
+    let mut work_product_decoder = Decoder::new(work_products_data, 0);
+    let work_products = try!(<Vec<SerializedWorkProduct>>::decode(&mut work_product_decoder));
+    reconcile_work_products(tcx, work_products, &dirty_nodes);
+
     Ok(())
 }
 
@@ -141,9 +171,9 @@ fn initial_dirty_nodes<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
         match hash.node.map_def(|&i| retraced.def_id(i)) {
             Some(dep_node) => {
                 let current_hash = hcx.hash(&dep_node).unwrap();
-                debug!("initial_dirty_nodes: hash of {:?} is {:?}, was {:?}",
-                       dep_node, current_hash, hash.hash);
                 if current_hash != hash.hash {
+                    debug!("initial_dirty_nodes: {:?} is dirty as hash is {:?}, was {:?}",
+                           dep_node, current_hash, hash.hash);
                     dirty_nodes.insert(dep_node);
                 }
             }
@@ -177,6 +207,8 @@ fn compute_clean_edges(serialized_edges: &[(SerializedEdge)],
                 clean_edges.push((source, target))
             } else {
                 // source removed, target must be dirty
+                debug!("compute_clean_edges: {:?} dirty because {:?} no longer exists",
+                       target, serialized_source);
                 dirty_nodes.insert(target);
             }
         } else {
@@ -213,3 +245,40 @@ fn compute_clean_edges(serialized_edges: &[(SerializedEdge)],
 
     clean_edges
 }
+
+/// Go through the list of work-products produced in the previous run.
+/// Delete any whose nodes have been found to be dirty or which are
+/// otherwise no longer applicable.
+fn reconcile_work_products<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
+                                     work_products: Vec<SerializedWorkProduct>,
+                                     dirty_nodes: &DirtyNodes) {
+    debug!("reconcile_work_products({:?})", work_products);
+    for swp in work_products {
+        let dep_node = DepNode::WorkProduct(swp.id.clone());
+        if dirty_nodes.contains(&dep_node) {
+            debug!("reconcile_work_products: dep-node for {:?} is dirty", swp);
+            delete_dirty_work_product(tcx, swp);
+        } else {
+            let path = in_incr_comp_dir(tcx.sess, &swp.work_product.file_name).unwrap();
+            if path.exists() {
+                tcx.dep_graph.insert_previous_work_product(&swp.id, swp.work_product);
+            } else {
+                debug!("reconcile_work_products: file for {:?} does not exist", swp);
+            }
+        }
+    }
+}
+
+fn delete_dirty_work_product(tcx: TyCtxt,
+                             swp: SerializedWorkProduct) {
+    debug!("delete_dirty_work_product({:?})", swp);
+    let path = in_incr_comp_dir(tcx.sess, &swp.work_product.file_name).unwrap();
+    match fs::remove_file(&path) {
+        Ok(()) => { }
+        Err(err) => {
+            tcx.sess.warn(
+                &format!("file-system error deleting outdated file `{}`: {}",
+                         path.display(), err));
+        }
+    }
+}
diff --git a/src/librustc_incremental/persist/mod.rs b/src/librustc_incremental/persist/mod.rs
@@ -22,3 +22,4 @@ mod util;
 
 pub use self::load::load_dep_graph;
 pub use self::save::save_dep_graph;
+pub use self::save::save_work_products;
diff --git a/src/librustc_incremental/persist/save.rs b/src/librustc_incremental/persist/save.rs
@@ -11,6 +11,7 @@
 use rbml::opaque::Encoder;
 use rustc::dep_graph::DepNode;
 use rustc::middle::cstore::LOCAL_CRATE;
+use rustc::session::Session;
 use rustc::ty::TyCtxt;
 use rustc_serialize::{Encodable as RustcEncodable};
 use std::hash::{Hasher, SipHasher};
@@ -24,19 +25,26 @@ use super::hash::*;
 use super::util::*;
 
 pub fn save_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>) {
+    debug!("save_dep_graph()");
     let _ignore = tcx.dep_graph.in_ignore();
+    let sess = tcx.sess;
     let mut hcx = HashContext::new(tcx);
-    save_in(&mut hcx, dep_graph_path(tcx), encode_dep_graph);
-    save_in(&mut hcx, metadata_hash_path(tcx, LOCAL_CRATE), encode_metadata_hashes);
+    save_in(sess, dep_graph_path(tcx), |e| encode_dep_graph(&mut hcx, e));
+    save_in(sess, metadata_hash_path(tcx, LOCAL_CRATE), |e| encode_metadata_hashes(&mut hcx, e));
 }
 
-fn save_in<'a, 'tcx, F>(hcx: &mut HashContext<'a, 'tcx>,
-                        opt_path_buf: Option<PathBuf>,
-                        encode: F)
-    where F: FnOnce(&mut HashContext<'a, 'tcx>, &mut Encoder) -> io::Result<()>
-{
-    let tcx = hcx.tcx;
+pub fn save_work_products(sess: &Session, local_crate_name: &str) {
+    debug!("save_work_products()");
+    let _ignore = sess.dep_graph.in_ignore();
+    let path = sess_work_products_path(sess, local_crate_name);
+    save_in(sess, path, |e| encode_work_products(sess, e));
+}
 
+fn save_in<F>(sess: &Session,
+              opt_path_buf: Option<PathBuf>,
+              encode: F)
+    where F: FnOnce(&mut Encoder) -> io::Result<()>
+{
     let path_buf = match opt_path_buf {
         Some(p) => p,
         None => return
@@ -49,7 +57,7 @@ fn save_in<'a, 'tcx, F>(hcx: &mut HashContext<'a, 'tcx>,
         match fs::remove_file(&path_buf) {
             Ok(()) => { }
             Err(err) => {
-                tcx.sess.err(
+                sess.err(
                     &format!("unable to delete old dep-graph at `{}`: {}",
                              path_buf.display(), err));
                 return;
@@ -59,10 +67,10 @@ fn save_in<'a, 'tcx, F>(hcx: &mut HashContext<'a, 'tcx>,
 
     // generate the data in a memory buffer
     let mut wr = Cursor::new(Vec::new());
-    match encode(hcx, &mut Encoder::new(&mut wr)) {
+    match encode(&mut Encoder::new(&mut wr)) {
         Ok(()) => { }
         Err(err) => {
-            tcx.sess.err(
+            sess.err(
                 &format!("could not encode dep-graph to `{}`: {}",
                          path_buf.display(), err));
             return;
@@ -77,7 +85,7 @@ fn save_in<'a, 'tcx, F>(hcx: &mut HashContext<'a, 'tcx>,
     {
         Ok(_) => { }
         Err(err) => {
-            tcx.sess.err(
+            sess.err(
                 &format!("failed to write dep-graph to `{}`: {}",
                          path_buf.display(), err));
             return;
@@ -192,3 +200,22 @@ pub fn encode_metadata_hashes<'a, 'tcx>(hcx: &mut HashContext<'a, 'tcx>,
 
     Ok(())
 }
+
+pub fn encode_work_products(sess: &Session,
+                            encoder: &mut Encoder)
+                            -> io::Result<()>
+{
+    let work_products: Vec<_> =
+        sess.dep_graph.work_products()
+                     .iter()
+                     .map(|(id, work_product)| {
+                         SerializedWorkProduct {
+                             id: id.clone(),
+                             work_product: work_product.clone(),
+                         }
+                     })
+                     .collect();
+
+    work_products.encode(encoder)
+}
+
diff --git a/src/librustc_incremental/persist/util.rs b/src/librustc_incremental/persist/util.rs

Original file line number	Diff line number	Diff line change
`@@ -22,3 +22,4 @@ mod util;`
`22`	`22`
`23`	`23`	`pub use self::load::load_dep_graph;`
`24`	`24`	`pub use self::save::save_dep_graph;`
	`25`	`+pub use self::save::save_work_products;`