---

yaml
---
r: 14504
b: refs/heads/try
c: b3f77bf
h: refs/heads/master
v: v3

Author: brson

[refs]

@@ -2,5 +2,5 @@
 refs/heads/master: 61b1875c16de39c166b0f4d54bba19f9c6777d1a
 refs/heads/snap-stage1: e33de59e47c5076a89eadeb38f4934f58a3618a6
 refs/heads/snap-stage3: 4a81779abd786ff22d71434c6d9a5917ea4cdfff
-refs/heads/try: e4c027446ec0d985be98dcbdce2b80308b88b12a
+refs/heads/try: b3f77bf92703543793a8073c8319e461e024cb69
 refs/tags/release-0.1: 1f5c5126e96c79d22cb7862f75304136e204f105

branches/try/src/rt/rust_kernel.cpp

@@ -20,7 +20,6 @@ rust_kernel::rust_kernel(rust_srv *srv) :
     live_tasks(0),
     max_task_id(0),
     rval(0),
-    live_schedulers(0),
     max_sched_id(0),
     env(srv->env)
 {
@@ -75,7 +74,6 @@ rust_kernel::create_scheduler(size_t num_threads) {
         bool is_new = sched_table
             .insert(std::pair<rust_sched_id, rust_scheduler*>(id, sched)).second;
         A(this, is_new, "Reusing a sched id?");
-        live_schedulers++;
     }
     sched->start_task_threads();
     return id;
@@ -97,26 +95,38 @@ void
 rust_kernel::release_scheduler_id(rust_sched_id id) {
     I(this, !sched_lock.lock_held_by_current_thread());
     scoped_lock with(sched_lock);
-    sched_map::iterator iter = sched_table.find(id);
-    I(this, iter != sched_table.end());
-    rust_scheduler *sched = iter->second;
-    sched_table.erase(iter);
-    delete sched;
-    live_schedulers--;
-    if (live_schedulers == 0) {
-        // We're all done. Tell the main thread to continue
-        sched_lock.signal();
-    }
-}
-
+    // This list will most likely only ever have a single element in it, but
+    // it's an actual list because we could potentially get here multiple
+    // times before the main thread ever calls wait_for_schedulers()
+    join_list.push_back(id);
+    sched_lock.signal();
+}
+
+/*
+Called on the main thread to wait for the kernel to exit. This function is
+also used to join on every terminating scheduler thread, so that we can be
+sure they have completely exited before the process exits.  If we don't join
+them then we can see valgrind errors due to un-freed pthread memory.
+ */
 int
 rust_kernel::wait_for_schedulers()
 {
     I(this, !sched_lock.lock_held_by_current_thread());
     scoped_lock with(sched_lock);
-    // Schedulers could possibly have already exited
-    if (live_schedulers != 0) {
-        sched_lock.wait();
+    while (!sched_table.empty()) {
+        while (!join_list.empty()) {
+            rust_sched_id id = join_list.back();
+            join_list.pop_back();
+            sched_map::iterator iter = sched_table.find(id);
+            I(this, iter != sched_table.end());
+            rust_scheduler *sched = iter->second;
+            sched_table.erase(iter);
+            sched->join_task_threads();
+            delete sched;
+        }
+        if (!sched_table.empty()) {
+            sched_lock.wait();
+        }
     }
     return rval;
 }

branches/try/src/rt/rust_kernel.h

@@ -3,6 +3,7 @@
 #define RUST_KERNEL_H
 
 #include <map>
+#include <vector>
 #include "memory_region.h"
 #include "rust_log.h"
 
@@ -36,14 +37,15 @@ class rust_kernel {
     lock_and_signal rval_lock;
     int rval;
 
-    // Protects live_schedulers, max_sched_id and sched_table
+    // Protects max_sched_id and sched_table, join_list
     lock_and_signal sched_lock;
-    // Tracks the number of schedulers currently running.
-    // When this hits 0 we will signal the sched_lock and the
-    // kernel will terminate.
-    uintptr_t live_schedulers;
+    // The next scheduler id
     rust_sched_id max_sched_id;
+    // A map from scheduler ids to schedulers. When this is empty
+    // the kernel terminates
     sched_map sched_table;
+    // A list of scheduler ids that are ready to exit
+    std::vector<rust_sched_id> join_list;
 
 public:
 

branches/try/src/rt/rust_scheduler.cpp

@@ -59,17 +59,21 @@ rust_scheduler::destroy_task_threads() {
 void
 rust_scheduler::start_task_threads()
 {
-    // Copy num_threads because it's possible for the last thread
-    // to terminate and have the kernel delete us before we
-    // hit the last check against num_threads, in which case
-    // we would be accessing invalid memory.
-    uintptr_t num_threads = this->num_threads;
     for(size_t i = 0; i < num_threads; ++i) {
         rust_task_thread *thread = threads[i];
         thread->start();
     }
 }
 
+void
+rust_scheduler::join_task_threads()
+{
+    for(size_t i = 0; i < num_threads; ++i) {
+        rust_task_thread *thread = threads[i];
+        thread->join();
+    }
+}
+
 void
 rust_scheduler::kill_all_tasks() {
     for(size_t i = 0; i < num_threads; ++i) {

branches/try/src/rt/rust_scheduler.h

@@ -37,6 +37,7 @@ class rust_scheduler : public kernel_owned<rust_scheduler> {
     ~rust_scheduler();
 
     void start_task_threads();
+    void join_task_threads();
     void kill_all_tasks();
     rust_task_id create_task(rust_task *spawner,
 			     const char *name,

branches/try/src/rt/rust_task_thread.cpp

@@ -320,7 +320,6 @@ rust_task_thread::create_task(rust_task *spawner, const char *name,
 
 void rust_task_thread::run() {
     this->start_main_loop();
-    detach();
     sched->release_task_thread();
 }