Guard pages + stack caching

src/libgreen/context.rs

@@ -12,10 +12,9 @@ use std::libc::c_void;
 use std::uint;
 use std::cast::{transmute, transmute_mut_unsafe,
                 transmute_region, transmute_mut_region};
+use stack::Stack;
 use std::unstable::stack;
 
-use stack::StackSegment;
-
 // FIXME #7761: Registers is boxed so that it is 16-byte aligned, for storing
 // SSE regs.  It would be marginally better not to do this. In C++ we
 // use an attribute on a struct.
@@ -41,7 +40,12 @@ impl Context {
     }
 
     /// Create a new context that will resume execution by running proc()
-    pub fn new(start: proc(), stack: &mut StackSegment) -> Context {
+    pub fn new(start: proc(), stack: &mut Stack) -> Context {
+        // FIXME #7767: Putting main into a ~ so it's a thin pointer and can
+        // be passed to the spawn function.  Another unfortunate
+        // allocation
+        let start = ~start;
+
         // The C-ABI function that is the task entry point
         //
         // Note that this function is a little sketchy. We're taking a
@@ -78,7 +82,6 @@ impl Context {
         // FIXME #7767: Putting main into a ~ so it's a thin pointer and can
         // be passed to the spawn function.  Another unfortunate
         // allocation
-        let start = ~start;
         initialize_call_frame(&mut *regs,
                               task_start_wrapper as *c_void,
                               unsafe { transmute(&*start) },

src/libgreen/coroutine.rs

@@ -14,7 +14,7 @@
 use std::rt::env;
 
 use context::Context;
-use stack::{StackPool, StackSegment};
+use stack::{StackPool, Stack};
 
 /// A coroutine is nothing more than a (register context, stack) pair.
 pub struct Coroutine {
@@ -24,7 +24,7 @@ pub struct Coroutine {
     ///
     /// Servo needs this to be public in order to tell SpiderMonkey
     /// about the stack bounds.
-    current_stack_segment: StackSegment,
+    current_stack_segment: Stack,
 
     /// Always valid if the task is alive and not running.
     saved_context: Context
@@ -39,7 +39,7 @@ impl Coroutine {
             Some(size) => size,
             None => env::min_stack()
         };
-        let mut stack = stack_pool.take_segment(stack_size);
+        let mut stack = stack_pool.take_stack(stack_size);
         let initial_context = Context::new(start, &mut stack);
         Coroutine {
             current_stack_segment: stack,
@@ -49,14 +49,14 @@ impl Coroutine {
 
     pub fn empty() -> Coroutine {
         Coroutine {
-            current_stack_segment: StackSegment::new(0),
+            current_stack_segment: Stack::new(0),
             saved_context: Context::empty()
         }
     }
 
     /// Destroy coroutine and try to reuse std::stack segment.
     pub fn recycle(self, stack_pool: &mut StackPool) {
         let Coroutine { current_stack_segment, .. } = self;
-        stack_pool.give_segment(current_stack_segment);
+        stack_pool.give_stack(current_stack_segment);
     }
 }

src/libgreen/stack.rs

@@ -8,46 +8,89 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use std::vec;
-use std::libc::{c_uint, uintptr_t};
+use std::rt::env::max_cached_stacks;
+use std::os::{errno, page_size, MemoryMap, MapReadable, MapWritable, MapNonStandardFlags};
+use std::libc::{MAP_STACK, MAP_PRIVATE, MAP_ANON};
+use std::libc::{c_uint, c_int, c_void, uintptr_t};
 
-pub struct StackSegment {
-    priv buf: ~[u8],
-    priv valgrind_id: c_uint
+/// A task's stack. The name "Stack" is a vestige of segmented stacks.
+pub struct Stack {
+    priv buf: MemoryMap,
+    priv min_size: uint,
+    priv valgrind_id: c_uint,
 }
 
-impl StackSegment {
-    pub fn new(size: uint) -> StackSegment {
-        unsafe {
-            // Crate a block of uninitialized values
-            let mut stack = vec::with_capacity(size);
-            stack.set_len(size);
+// This is what glibc uses on linux. MAP_STACK can be 0 if any platform we decide to support doesn't
+// have it defined. Considering also using MAP_GROWSDOWN
+static STACK_FLAGS: c_int = MAP_STACK | MAP_PRIVATE | MAP_ANON;
 
-            let mut stk = StackSegment {
-                buf: stack,
-                valgrind_id: 0
-            };
+impl Stack {
+    pub fn new(size: uint) -> Stack {
+        // Map in a stack. Eventually we might be able to handle stack allocation failure, which
+        // would fail to spawn the task. But there's not many sensible things to do on OOM.
+        // Failure seems fine (and is what the old stack allocation did).
+        let stack = MemoryMap::new(size, [MapReadable, MapWritable,
+                                          MapNonStandardFlags(STACK_FLAGS)]).unwrap();
 
-            // XXX: Using the FFI to call a C macro. Slow
-            stk.valgrind_id = rust_valgrind_stack_register(stk.start(), stk.end());
-            return stk;
+        // Change the last page to be inaccessible. This is to provide safety; when an FFI
+        // function overflows it will (hopefully) hit this guard page. It isn't guaranteed, but
+        // that's why FFI is unsafe. buf.data is guaranteed to be aligned properly.
+        if !protect_last_page(&stack) {
+            fail!("Could not memory-protect guard page. stack={:?}, errno={}",
+                  stack, errno());
         }
+
+        let mut stk = Stack {
+            buf: stack,
+            min_size: size,
+            valgrind_id: 0
+        };
+
+        // XXX: Using the FFI to call a C macro. Slow
+        stk.valgrind_id = unsafe { rust_valgrind_stack_register(stk.start(), stk.end()) };
+        return stk;
     }
 
     /// Point to the low end of the allocated stack
     pub fn start(&self) -> *uint {
-        self.buf.as_ptr() as *uint
+        self.buf.data as *uint
     }
 
     /// Point one word beyond the high end of the allocated stack
     pub fn end(&self) -> *uint {
         unsafe {
-            self.buf.as_ptr().offset(self.buf.len() as int) as *uint
+            self.buf.data.offset(self.buf.len as int) as *uint
         }
     }
 }
 
-impl Drop for StackSegment {
+// These use ToPrimitive so that we never need to worry about the sizes of whatever types these
+// (which we would with scalar casts). It's either a wrapper for a scalar cast or failure: fast, or
+// will fail during compilation.
+#[cfg(unix)]
+fn protect_last_page(stack: &MemoryMap) -> bool {
+    use std::libc::{mprotect, PROT_NONE, size_t};
+    unsafe {
+        // This may seem backwards: the start of the segment is the last page? Yes! The stack grows
+        // from higher addresses (the end of the allocated block) to lower addresses (the start of
+        // the allocated block).
+        let last_page = stack.data as *c_void;
+        mprotect(last_page, page_size() as size_t, PROT_NONE) != -1
+    }
+}
+
+#[cfg(windows)]
+fn protect_last_page(stack: &MemoryMap) -> bool {
+    use std::libc::{VirtualProtect, PAGE_NOACCESS, SIZE_T, LPDWORD, DWORD};
+    unsafe {
+        // see above
+        let last_page = stack.data as *c_void;
+        let old_prot: DWORD = 0;
+        VirtualProtect(last_page, page_size() as SIZE_T, PAGE_NOACCESS, &old_prot as LPDWORD) != 0
+    }
+}
+
+impl Drop for Stack {
     fn drop(&mut self) {
         unsafe {
             // XXX: Using the FFI to call a C macro. Slow
@@ -56,16 +99,30 @@ impl Drop for StackSegment {
     }
 }
 
-pub struct StackPool(());
+pub struct StackPool {
+    // Ideally this would be some datastructure that preserved ordering on Stack.min_size.
+    priv stacks: ~[Stack],
+}
 
 impl StackPool {
-    pub fn new() -> StackPool { StackPool(()) }
+    pub fn new() -> StackPool {
+        StackPool {
+            stacks: ~[],
+        }
+    }
 
-    pub fn take_segment(&self, min_size: uint) -> StackSegment {
-        StackSegment::new(min_size)
+    pub fn take_stack(&mut self, min_size: uint) -> Stack {
+        // Ideally this would be a binary search
+        match self.stacks.iter().position(|s| s.min_size < min_size) {
+            Some(idx) => self.stacks.swap_remove(idx),
+            None      => Stack::new(min_size)
+        }
     }
 
-    pub fn give_segment(&self, _stack: StackSegment) {
+    pub fn give_stack(&mut self, stack: Stack) {
+        if self.stacks.len() <= max_cached_stacks() {
+            self.stacks.push(stack)
+        }
     }
 }
 

src/libstd/libc.rs

@@ -2823,6 +2823,7 @@ pub mod consts {
             pub static MAP_PRIVATE : c_int = 0x0002;
             pub static MAP_FIXED : c_int = 0x0010;
             pub static MAP_ANON : c_int = 0x1000;
+            pub static MAP_STACK : c_int = 0;
 
             pub static MAP_FAILED : *c_void = -1 as *c_void;
 

src/libstd/os.rs

@@ -843,8 +843,8 @@ pub enum MemoryMapKind {
     /// Memory-mapped file. On Windows, the inner pointer is a handle to the mapping, and
     /// corresponds to `CreateFileMapping`. Elsewhere, it is null.
     MapFile(*c_void),
-    /// Virtual memory map. Usually used to change the permissions of a given chunk of memory.
-    /// Corresponds to `VirtualAlloc` on Windows.
+    /// Virtual memory map. Usually used to change the permissions of a given chunk of memory, or
+    /// for allocation. Corresponds to `VirtualAlloc` on Windows.
     MapVirtual
 }
 
@@ -861,7 +861,11 @@ pub enum MapOption {
     /// Create a memory mapping for a file with a given fd.
     MapFd(c_int),
     /// When using `MapFd`, the start of the map is `uint` bytes from the start of the file.
-    MapOffset(uint)
+    MapOffset(uint),
+    /// On POSIX, this can be used to specify the default flags passed to `mmap`. By default it uses
+    /// `MAP_PRIVATE` and, if not using `MapFd`, `MAP_ANON`. This will override both of those. This
+    /// is platform-specific (the exact values used) and ignored on Windows.
+    MapNonStandardFlags(c_int),
 }
 
 /// Possible errors when creating a map.
@@ -931,6 +935,7 @@ impl MemoryMap {
         let mut flags: c_int = libc::MAP_PRIVATE;
         let mut fd: c_int = -1;
         let mut offset: off_t = 0;
+        let mut custom_flags = false;
         let len = round_up(min_len, page_size()) as size_t;
 
         for &o in options.iter() {
@@ -946,10 +951,11 @@ impl MemoryMap {
                     flags |= libc::MAP_FILE;
                     fd = fd_;
                 },
-                MapOffset(offset_) => { offset = offset_ as off_t; }
+                MapOffset(offset_) => { offset = offset_ as off_t; },
+                MapNonStandardFlags(f) => { custom_flags = true; flags = f },
             }
         }
-        if fd == -1 { flags |= libc::MAP_ANON; }
+        if fd == -1 && !custom_flags { flags |= libc::MAP_ANON; }
 
         let r = unsafe {
             libc::mmap(addr, len, prot, flags, fd, offset)
@@ -1020,7 +1026,9 @@ impl MemoryMap {
                 MapExecutable => { executable = true; }
                 MapAddr(addr_) => { lpAddress = addr_ as LPVOID; },
                 MapFd(fd_) => { fd = fd_; },
-                MapOffset(offset_) => { offset = offset_; }
+                MapOffset(offset_) => { offset = offset_; },
+                MapNonStandardFlags(f) => info!("MemoryMap::new: MapNonStandardFlags used on \
+                                                Windows: {}", f),
             }
         }
 

src/libstd/rt/env.rs

@@ -10,26 +10,33 @@
 
 //! Runtime environment settings
 
-use from_str::FromStr;
+use from_str::from_str;
 use option::{Some, None};
 use os;
 
 // Note that these are all accessed without any synchronization.
 // They are expected to be initialized once then left alone.
 
 static mut MIN_STACK: uint = 2 * 1024 * 1024;
+/// This default corresponds to 20M of cache per scheduler (at the default size).
+static mut MAX_CACHED_STACKS: uint = 10;
 static mut DEBUG_BORROW: bool = false;
 static mut POISON_ON_FREE: bool = false;
 
 pub fn init() {
     unsafe {
         match os::getenv("RUST_MIN_STACK") {
-            Some(s) => match FromStr::from_str(s) {
+            Some(s) => match from_str(s) {
                 Some(i) => MIN_STACK = i,
                 None => ()
             },
             None => ()
         }
+        match os::getenv("RUST_MAX_CACHED_STACKS") {
+            Some(max) => MAX_CACHED_STACKS = from_str(max).expect("expected positive integer in \
+                                                                   RUST_MAX_CACHED_STACKS"),
+            None => ()
+        }
         match os::getenv("RUST_DEBUG_BORROW") {
             Some(_) => DEBUG_BORROW = true,
             None => ()
@@ -45,6 +52,10 @@ pub fn min_stack() -> uint {
     unsafe { MIN_STACK }
 }
 
+pub fn max_cached_stacks() -> uint {
+    unsafe { MAX_CACHED_STACKS }
+}
+
 pub fn debug_borrow() -> bool {
     unsafe { DEBUG_BORROW }
 }

src/test/bench/silly-test-spawn.rs

@@ -0,0 +1,16 @@
+// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// Useful smoketest for scheduler performance.
+fn main() {
+    for _ in range(1, 100_000) {
+        do spawn { }
+    }
+}

src/test/bench/spawnone.rs

@@ -0,0 +1,14 @@
+// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// Useful for checking syscall usage of baseline scheduler usage
+fn main() {
+    do spawn { }
+}

src/test/bench/task-perf-one-million.rs

@@ -56,7 +56,7 @@ fn main() {
         args
     };
 
-    let children = from_str::<uint>(args[1]).get();
+    let children = from_str::<uint>(args[1]).unwrap();
     let (wait_port, wait_chan) = stream();
     do task::spawn {
         calc(children, &wait_chan);
@@ -66,5 +66,5 @@ fn main() {
     let (sum_port, sum_chan) = stream::<int>();
     start_chan.send(sum_chan);
     let sum = sum_port.recv();
-    error!("How many tasks? %d tasks.", sum);
+    error!("How many tasks? {} tasks.", sum);
 }