Add a 10 minute timeout to individual tests.

On the bots, we have a timeout without output of 60 minutes for the entire test.
This should ensure that we are able to kill mis-behaving tests and give a good
error instead of just getting a jenkins timeout error.

For those confused, this is for the guard malloc/leaks test.

rdar://36874229

Author: gottesmm

benchmark/scripts/Benchmark_RuntimeLeaksRunner.in

@@ -81,29 +81,35 @@ class LeaksRunnerBenchmarkDriver(perf_test_driver.BenchmarkDriver):
     def prepare_input(self, name):
         return {'num_samples': self.num_samples, 'num_iters': self.num_iters}
 
+    def run_test_inner(self, data, num_iters):
+        p = subprocess.Popen([
+            data['path'],
+            "--num-samples={}".format(data['num_samples']),
+            "--num-iters={}".format(num_iters), data['test_name']],
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        error_out = p.communicate()[1].split("\n")
+        result = p.returncode
+        if result is None:
+            raise RuntimeError("Expected one line of output")
+        if result != 0:
+            raise RuntimeError("Process segfaulted")
+        return error_out
+
     def run_test(self, data, num_iters):
         try:
-            p = subprocess.Popen([
-                data['path'],
-                "--num-samples={}".format(data['num_samples']),
-                "--num-iters={}".format(num_iters), data['test_name']],
-                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            error_out = p.communicate()[1].split("\n")
-            result = p.returncode
-            if result is None:
-                raise RuntimeError("Expected one line of output")
-            if result != 0:
-                raise RuntimeError("Process segfaulted")
-        except Exception:
-            sys.stderr.write("Child Process Failed! (%s,%s)\n" % (
-                data['path'], data['test_name']))
+            args = [data, num_iters]
+            result = perf_test_driver.run_with_timeout(self.run_test_inner,
+                                                       args)
+        except Exception, e:
+            sys.stderr.write("Child Process Failed! (%s,%s). Error: %s\n" % (
+                data['path'], data['test_name'], e))
             sys.stderr.flush()
             return None
 
         try:
             # We grab the second line since swift globals get lazily created in
             # the first iteration.
-            d = json.loads(error_out[1])
+            d = json.loads(result[1])
             d['objc_objects'] = [x for x in d['objc_objects']
                                  if x not in IGNORABLE_GLOBAL_OBJC_CLASSES]
             d['objc_count'] = len(d['objc_objects'])

benchmark/scripts/perf_test_driver/perf_test_driver.py

@@ -57,6 +57,24 @@ def print_data(self, max_test_len):
         print(fmt.format(self.get_name(), self.get_result()))
 
 
+def run_with_timeout(func, args):
+    # We timeout after 10 minutes.
+    timeout_seconds = 10 * 60
+
+    # We just use this to create a timeout since we use an older python. Once
+    # we update to use python >= 3.3, use the timeout API on communicate
+    # instead.
+    import multiprocessing.dummy
+    fakeThreadPool = multiprocessing.dummy.Pool(1)
+    try:
+        result = fakeThreadPool.apply_async(func, args=args)
+        return result.get(timeout_seconds)
+    except multiprocessing.TimeoutError:
+        fakeThreadPool.terminate()
+        raise RuntimeError("Child process aborted due to timeout. "
+                           "Timeout: %s seconds" % timeout_seconds)
+
+
 def _unwrap_self(args):
     return type(args[0]).process_input(*args)