progressive linking (#564)

* Implement progressive linking

* Address feedback

Author: maxded

src/lib.rs

@@ -1774,66 +1774,25 @@ impl Build {
     }
 
     fn assemble(&self, lib_name: &str, dst: &Path, objs: &[Object]) -> Result<(), Error> {
-        // Delete the destination if it exists as the `ar` tool at least on Unix
-        // appends to it, which we don't want.
+        // Delete the destination if it exists as we want to
+        // create on the first iteration instead of appending.
         let _ = fs::remove_file(&dst);
-
-        let objects: Vec<_> = objs.iter().map(|obj| obj.dst.clone()).collect();
         let target = self.get_target()?;
-        if target.contains("msvc") {
-            let (mut cmd, program) = self.get_ar()?;
-            let mut out = OsString::from("-out:");
-            out.push(dst);
-            cmd.arg(out).arg("-nologo");
-            for flag in self.ar_flags.iter() {
-                cmd.arg(flag);
-            }
-
-            // Similar to https://github.com/rust-lang/rust/pull/47507
-            // and https://github.com/rust-lang/rust/pull/48548
-            let estimated_command_line_len = objects
-                .iter()
-                .chain(&self.objects)
-                .map(|a| a.as_os_str().len())
-                .sum::<usize>();
-            if estimated_command_line_len > 1024 * 6 {
-                let mut args = String::from("\u{FEFF}"); // BOM
-                for arg in objects.iter().chain(&self.objects) {
-                    args.push('"');
-                    for c in arg.to_str().unwrap().chars() {
-                        if c == '"' {
-                            args.push('\\')
-                        }
-                        args.push(c)
-                    }
-                    args.push('"');
-                    args.push('\n');
-                }
-
-                let mut utf16le = Vec::new();
-                for code_unit in args.encode_utf16() {
-                    utf16le.push(code_unit as u8);
-                    utf16le.push((code_unit >> 8) as u8);
-                }
-
-                let mut args_file = OsString::from(dst);
-                args_file.push(".args");
-                fs::File::create(&args_file)
-                    .unwrap()
-                    .write_all(&utf16le)
-                    .unwrap();
+        let objs: Vec<_> = objs
+            .iter()
+            .map(|o| o.dst.clone())
+            .chain(self.objects.clone())
+            .collect();
 
-                let mut args_file_arg = OsString::from("@");
-                args_file_arg.push(args_file);
-                cmd.arg(args_file_arg);
-            } else {
-                cmd.args(&objects).args(&self.objects);
-            }
-            run(&mut cmd, &program)?;
+        for chunk in objs.chunks(100) {
+            self.assemble_progressive(dst, chunk)?;
+        }
 
+        if target.contains("msvc") {
             // The Rust compiler will look for libfoo.a and foo.lib, but the
             // MSVC linker will also be passed foo.lib, so be sure that both
             // exist for now.
+
             let lib_dst = dst.with_file_name(format!("{}.lib", lib_name));
             let _ = fs::remove_file(&lib_dst);
             match fs::hard_link(&dst, &lib_dst).or_else(|_| {
@@ -1848,6 +1807,29 @@ impl Build {
                     ));
                 }
             };
+        }
+
+        Ok(())
+    }
+
+    fn assemble_progressive(&self, dst: &Path, objs: &[PathBuf]) -> Result<(), Error> {
+        let target = self.get_target()?;
+
+        if target.contains("msvc") {
+            let (mut cmd, program) = self.get_ar()?;
+            let mut out = OsString::from("-out:");
+            out.push(dst);
+            cmd.arg(out).arg("-nologo");
+            for flag in self.ar_flags.iter() {
+                cmd.arg(flag);
+            }
+            // If the library file already exists, add the libary name
+            // as an argument to let lib.exe know we are appending the objs.
+            if dst.exists() {
+                cmd.arg(dst);
+            }
+            cmd.args(objs);
+            run(&mut cmd, &program)?;
         } else {
             let (mut ar, cmd) = self.get_ar()?;
 
@@ -1877,10 +1859,7 @@ impl Build {
             for flag in self.ar_flags.iter() {
                 ar.arg(flag);
             }
-            run(
-                ar.arg("crs").arg(dst).args(&objects).args(&self.objects),
-                &cmd,
-            )?;
+            run(ar.arg("crs").arg(dst).args(objs), &cmd)?;
         }
 
         Ok(())