Starting to implement a frame allocator for Rustc

Author: mattico

configure

@@ -637,6 +637,7 @@ opt codegen-tests 1 "run the src/test/codegen tests"
 opt option-checking 1 "complain about unrecognized options in this configure script"
 opt ninja 0 "build LLVM using the Ninja generator (for MSVC, requires building in the correct environment)"
 opt vendor 0 "enable usage of vendored Rust crates"
+opt frame-alloc 0 "enable the frame allocator"
 
 # Optimization and debugging options. These may be overridden by the release channel, etc.
 opt_nosave optimize 1 "build optimized rust code"

src/Cargo.lock

@@ -77,6 +77,7 @@ pub struct Config {
     // libstd features
     pub debug_jemalloc: bool,
     pub use_jemalloc: bool,
+    pub use_alloc_frame: bool,
     pub backtrace: bool, // support for RUST_BACKTRACE
 
     // misc
@@ -176,6 +177,7 @@ struct Rust {
     debuginfo_lines: Option<bool>,
     debug_jemalloc: Option<bool>,
     use_jemalloc: Option<bool>,
+    use_alloc_frame: Option<bool>,
     backtrace: Option<bool>,
     default_linker: Option<String>,
     default_ar: Option<String>,
@@ -203,6 +205,7 @@ impl Config {
         let mut config = Config::default();
         config.llvm_optimize = true;
         config.use_jemalloc = true;
+        config.use_alloc_frame = false;
         config.backtrace = true;
         config.rust_optimize = true;
         config.rust_optimize_tests = true;
@@ -298,6 +301,7 @@ impl Config {
             set(&mut config.rust_rpath, rust.rpath);
             set(&mut config.debug_jemalloc, rust.debug_jemalloc);
             set(&mut config.use_jemalloc, rust.use_jemalloc);
+            set(&mut config.use_alloc_frame, rust.use_alloc_frame);
             set(&mut config.backtrace, rust.backtrace);
             set(&mut config.channel, rust.channel.clone());
             config.rustc_default_linker = rust.default_linker.clone();
@@ -385,6 +389,7 @@ impl Config {
                 ("DEBUGINFO_LINES", self.rust_debuginfo_lines),
                 ("JEMALLOC", self.use_jemalloc),
                 ("DEBUG_JEMALLOC", self.debug_jemalloc),
+                ("FRAME_ALLOC", self.use_alloc_frame),
                 ("RPATH", self.rust_rpath),
                 ("OPTIMIZE_TESTS", self.rust_optimize_tests),
                 ("DEBUGINFO_TESTS", self.rust_debuginfo_tests),

src/bootstrap/config.rs

@@ -137,6 +137,9 @@
 # Whether or not jemalloc is built with its debug option set
 #debug-jemalloc = false
 
+# Whether or not the frame allocator is built and enabled
+#use-alloc-frame = false
+
 # Whether or not `panic!`s generate backtraces (RUST_BACKTRACE)
 #backtrace = true
 

src/bootstrap/config.toml.example

@@ -609,6 +609,9 @@ impl Build {
         if self.config.use_jemalloc {
             features.push_str(" jemalloc");
         }
+        if self.config.use_alloc_frame {
+            features.push_str(" alloc_frame");
+        }
         if self.config.backtrace {
             features.push_str(" backtrace");
         }
@@ -621,6 +624,9 @@ impl Build {
         if self.config.use_jemalloc {
             features.push_str(" jemalloc");
         }
+        if self.config.use_alloc_frame {
+            features.push_str(" alloc_frame");
+        }
         return features
     }
 

src/bootstrap/lib.rs

@@ -0,0 +1,13 @@
+[package]
+authors = ["The Rust Project Developers"]
+name = "alloc_frame"
+version = "0.0.0"
+
+[lib]
+name = "alloc_frame"
+path = "lib.rs"
+test = false
+
+[dependencies]
+core = { path = "../libcore" }
+libc = { path = "../rustc/libc_shim" }

src/liballoc_frame/Cargo.toml

@@ -0,0 +1,191 @@
+// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#![crate_name = "alloc_frame"]
+#![crate_type = "rlib"]
+#![no_std]
+#![allocator]
+#![cfg_attr(not(stage0), deny(warnings))]
+#![unstable(feature = "alloc_frame",
+            reason = "this library is unlikely to be stabilized in its current \
+                      form or name",
+            issue = "0")]
+#![feature(allocator)]
+#![feature(const_fn)]
+#![feature(staged_api)]
+#![feature(libc)]
+#![cfg_attr(any(unix, target_os = "redox"), feature(libc))]
+
+extern crate libc;
+
+use core::ptr;
+
+// The minimum alignment guaranteed by the architecture. This value is used to
+// add fast paths for low alignment values. In practice, the alignment is a
+// constant at the call site and the branch will be optimized out.
+#[cfg(all(any(target_arch = "x86",
+              target_arch = "arm",
+              target_arch = "mips",
+              target_arch = "powerpc",
+              target_arch = "powerpc64",
+              target_arch = "asmjs",
+              target_arch = "wasm32")))]
+const MIN_ALIGN: usize = 8;
+#[cfg(all(any(target_arch = "x86_64",
+              target_arch = "aarch64",
+              target_arch = "mips64",
+              target_arch = "s390x")))]
+const MIN_ALIGN: usize = 16;
+
+// The size of the chunk which is actually allocated
+const CHUNK_SIZE: usize = 4096 * 16;
+const CHUNK_ALIGN: usize = 4096;
+
+static mut HEAP: *mut u8 = ptr::null_mut();
+static mut HEAP_LEFT: usize = 0;
+
+#[no_mangle]
+pub extern "C" fn __rust_allocate(size: usize, align: usize) -> *mut u8 {
+    let new_align = if align < MIN_ALIGN { MIN_ALIGN } else { align };
+    let new_size = (size + new_align - 1) & !(new_align - 1);
+
+    unsafe {
+        if new_size < HEAP_LEFT {
+            HEAP_LEFT -= new_size;
+            let p = HEAP;
+            HEAP = HEAP.offset(new_size as isize);
+            return p;
+        } else if new_size > CHUNK_SIZE {
+            return imp::allocate(size, align);
+        } else {
+            HEAP_LEFT = CHUNK_SIZE - new_size;
+            let p = imp::allocate(CHUNK_SIZE, CHUNK_ALIGN);
+            HEAP = p.offset(new_size as isize);
+            return p;
+        }
+    }
+}
+
+#[no_mangle]
+pub extern "C" fn __rust_deallocate(_ptr: *mut u8, _old_size: usize, _align: usize) {
+}
+
+#[no_mangle]
+pub extern "C" fn __rust_reallocate(ptr: *mut u8,
+                                    old_size: usize,
+                                    size: usize,
+                                    align: usize)
+                                    -> *mut u8 {
+    let new_ptr = __rust_allocate(size, align);
+    unsafe { libc::memcpy(new_ptr as *mut _, ptr as *mut _, old_size); } 
+    new_ptr
+}
+
+#[no_mangle]
+pub extern "C" fn __rust_reallocate_inplace(_ptr: *mut u8,
+                                            old_size: usize,
+                                            _size: usize,
+                                            _align: usize)
+                                            -> usize {
+    old_size
+}
+
+#[no_mangle]
+pub extern "C" fn __rust_usable_size(size: usize, _align: usize) -> usize {
+    size
+}
+
+#[cfg(any(unix, target_os = "redox"))]
+mod imp {
+    use core::cmp;
+    use MIN_ALIGN;
+
+    pub unsafe fn allocate(size: usize, align: usize) -> *mut u8 {
+        if align <= MIN_ALIGN {
+            libc::malloc(size as libc::size_t) as *mut u8
+        } else {
+            aligned_malloc(size, align)
+        }
+    }
+
+    #[cfg(any(target_os = "android", target_os = "redox"))]
+    unsafe fn aligned_malloc(size: usize, align: usize) -> *mut u8 {
+        // On android we currently target API level 9 which unfortunately
+        // doesn't have the `posix_memalign` API used below. Instead we use
+        // `memalign`, but this unfortunately has the property on some systems
+        // where the memory returned cannot be deallocated by `free`!
+        //
+        // Upon closer inspection, however, this appears to work just fine with
+        // Android, so for this platform we should be fine to call `memalign`
+        // (which is present in API level 9). Some helpful references could
+        // possibly be chromium using memalign [1], attempts at documenting that
+        // memalign + free is ok [2] [3], or the current source of chromium
+        // which still uses memalign on android [4].
+        //
+        // [1]: https://codereview.chromium.org/10796020/
+        // [2]: https://code.google.com/p/android/issues/detail?id=35391
+        // [3]: https://bugs.chromium.org/p/chromium/issues/detail?id=138579
+        // [4]: https://chromium.googlesource.com/chromium/src/base/+/master/
+        //                                       /memory/aligned_memory.cc
+        libc::memalign(align as libc::size_t, size as libc::size_t) as *mut u8
+    }
+
+    #[cfg(not(any(target_os = "android", target_os = "redox")))]
+    unsafe fn aligned_malloc(size: usize, align: usize) -> *mut u8 {
+        let mut out = ptr::null_mut();
+        let ret = libc::posix_memalign(&mut out, align as libc::size_t, size as libc::size_t);
+        if ret != 0 {
+            ptr::null_mut()
+        } else {
+            out as *mut u8
+        }
+    }
+}
+
+#[cfg(windows)]
+#[allow(bad_style)]
+mod imp {
+    use MIN_ALIGN;
+
+    type LPVOID = *mut u8;
+    type HANDLE = LPVOID;
+    type SIZE_T = usize;
+    type DWORD = u32;
+
+    extern "system" {
+        fn GetProcessHeap() -> HANDLE;
+        fn HeapAlloc(hHeap: HANDLE, dwFlags: DWORD, dwBytes: SIZE_T) -> LPVOID;
+    }
+
+    #[repr(C)]
+    struct Header(*mut u8);
+
+    unsafe fn get_header<'a>(ptr: *mut u8) -> &'a mut Header {
+        &mut *(ptr as *mut Header).offset(-1)
+    }
+
+    unsafe fn align_ptr(ptr: *mut u8, align: usize) -> *mut u8 {
+        let aligned = ptr.offset((align - (ptr as usize & (align - 1))) as isize);
+        *get_header(aligned) = Header(ptr);
+        aligned
+    }
+
+    pub unsafe fn allocate(size: usize, align: usize) -> *mut u8 {
+        if align <= MIN_ALIGN {
+            HeapAlloc(GetProcessHeap(), 0, size as SIZE_T) as *mut u8
+        } else {
+            let ptr = HeapAlloc(GetProcessHeap(), 0, (size + align) as SIZE_T) as *mut u8;
+            if ptr.is_null() {
+                return ptr;
+            }
+            align_ptr(ptr, align)
+        }
+    }
+}

src/liballoc_frame/lib.rs

@@ -13,6 +13,7 @@ crate-type = ["dylib", "rlib"]
 alloc = { path = "../liballoc" }
 alloc_jemalloc = { path = "../liballoc_jemalloc", optional = true }
 alloc_system = { path = "../liballoc_system" }
+alloc_frame = { path = "../liballoc_frame", optional = true }
 panic_unwind = { path = "../libpanic_unwind", optional = true }
 panic_abort = { path = "../libpanic_abort" }
 collections = { path = "../libcollections" }

src/libstd/Cargo.toml

@@ -331,7 +331,7 @@ extern crate libc;
 // We always need an unwinder currently for backtraces
 extern crate unwind;
 
-#[cfg(stage0)]
+#[cfg(all(stage0, not(alloc_frame)))]
 extern crate alloc_system;
 
 // compiler-rt intrinsics

src/libstd/lib.rs

@@ -17,6 +17,7 @@ path = "rustdoc.rs"
 rustc_back = { path = "../librustc_back" }
 rustc_driver = { path = "../librustc_driver" }
 rustdoc = { path = "../librustdoc" }
+alloc_frame = { path = "../liballoc_frame", optional = true }
 
 [features]
 jemalloc = ["rustc_back/jemalloc"]

src/rustc/Cargo.toml

@@ -9,7 +9,12 @@
 // except according to those terms.
 
 #![feature(rustc_private)]
+#![cfg_attr(feature = "alloc_frame", feature(alloc_frame))]
 
 extern crate rustc_driver;
 
+// Use the frame allocator to speed up runtime
+#[cfg(feature = "alloc_frame")]
+extern crate alloc_frame;
+
 fn main() { rustc_driver::main() }

src/rustc/rustc.rs

@@ -9,7 +9,12 @@
 // except according to those terms.
 
 #![feature(rustdoc)]
+#![cfg_attr(feature = "alloc_frame", feature(alloc_frame))]
 
 extern crate rustdoc;
 
+// Use the frame allocator to speed up runtime
+#[cfg(feature = "alloc_frame")]
+extern crate alloc_frame;
+
 fn main() { rustdoc::main() }

src/rustc/rustdoc.rs

@@ -37,4 +37,5 @@ core = { path = "../../libcore" }
 backtrace = ["std/backtrace"]
 debug-jemalloc = ["std/debug-jemalloc"]
 jemalloc = ["std/jemalloc"]
+alloc_frame = ["std/alloc_frame"]
 panic-unwind = ["std/panic-unwind"]

src/rustc/std_shim/Cargo.toml

@@ -53,6 +53,7 @@ const EXCEPTION_PATHS: &'static [&'static str] = &[
     // std crates
     "src/liballoc_jemalloc",
     "src/liballoc_system",
+    "src/liballoc_frame",
     "src/liblibc",
     "src/libpanic_abort",
     "src/libpanic_unwind",