Parallelize unit tests on CI server (Travis) (#1251)

Fixes #1247
Temporary work around for #1250 (didn't add a news entry for the work around)

Author: DonJayamanne

.travis.yml

@@ -5,8 +5,22 @@ matrix:
     include:
         - os: linux
           python: "2.7"
+          env: DEBUGGER_TEST=true
+        - os: linux
+          python: "2.7"
+          env: SINGLE_WORKSPACE_TEST=true
+        - os: linux
+          python: "2.7"
+          env: MULTIROOT_WORKSPACE_TEST=true
+        - os: linux
+          python: "3.6-dev"
+          env: DEBUGGER_TEST=true
         - os: linux
           python: "3.6-dev"
+          env: SINGLE_WORKSPACE_TEST=true
+        - os: linux
+          python: "3.6-dev"
+          env: MULTIROOT_WORKSPACE_TEST=true
 before_install: |
   if [ $TRAVIS_OS_NAME == "linux" ]; then
     export CXX="g++-4.9" CC="gcc-4.9" DISPLAY=:99.0;
@@ -30,7 +44,9 @@ script:
   - yarn run clean
   - yarn run vscode:prepublish
   - yarn run cover:enable
-  - yarn run testDebugger --silent
+  - if [ $DEBUGGER_TEST == "true" ]; then
+        yarn run testDebugger --silent;
+    fi
   - yarn run debugger-coverage
   - if [ $TRAVIS_UPLOAD_COVERAGE == "true" ]; then
         bash <(curl -s https://codecov.io/bash);
@@ -44,14 +60,18 @@ script:
   - yarn run clean
   - yarn run vscode:prepublish
   - yarn run cover:enable
-  - yarn run testSingleWorkspace --silent
+  - if [ $SINGLE_WORKSPACE_TEST == "true" ]; then
+        yarn run testSingleWorkspace --silent;
+    fi
   - if [ $TRAVIS_UPLOAD_COVERAGE == "true" ]; then
         bash <(curl -s https://codecov.io/bash);
     fi
   - yarn run clean
   - yarn run vscode:prepublish
   - yarn run cover:enable
-  - yarn run testMultiWorkspace --silent
+  - if [ $MULTIROOT_WORKSPACE_TEST == "true" ]; then
+        yarn run testMultiWorkspace --silent;
+    fi
   - if [ $TRAVIS_UPLOAD_COVERAGE == "true" ]; then
         bash <(curl -s https://codecov.io/bash);
     fi

appveyor.yml

@@ -8,6 +8,25 @@ environment:
       PYTHON_ARCH: "32"
       nodejs_version: "8.9.1"
       APPVEYOR: "true"
+      DEBUGGER_TEST: "true"
+    - PYTHON: "C:\\Python36"
+      PYTHON_VERSION: "3.6.3"
+      PYTHON_ARCH: "32"
+      nodejs_version: "8.9.1"
+      APPVEYOR: "true"
+      SINGLE_WORKSPACE_TEST: "true"
+    - PYTHON: "C:\\Python36"
+      PYTHON_VERSION: "3.6.3"
+      PYTHON_ARCH: "32"
+      nodejs_version: "8.9.1"
+      APPVEYOR: "true"
+      MULTIROOT_WORKSPACE_TEST: "true"
+    - PYTHON: "C:\\Python36"
+      PYTHON_VERSION: "3.6.3"
+      PYTHON_ARCH: "32"
+      nodejs_version: "8.9.1"
+      APPVEYOR: "true"
+      ANALYSIS_TEST: "true"
 
 init:
   - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%"
@@ -37,9 +56,11 @@ build: off
 test_script:
   - yarn run clean
   - yarn run vscode:prepublish
-  - yarn run testDebugger --silent
-  - yarn run testSingleWorkspace --silent
-  - yarn run testMultiWorkspace --silent
-  # - yarn run testAnalysisEngine --silent
-
-
+  - if [%DEBUGGER_TEST%]==[true] (
+      yarn run testDebugger --silent)
+  - if [%SINGLE_WORKSPACE_TEST%]==[true] (
+      yarn run testSingleWorkspace --silent)
+  - if [%MULTIROOT_WORKSPACE_TEST%]==[true] (
+      yarn run testMultiWorkspace --silent)
+  # - if [%ANALYSIS_TEST%]==[true] (
+  #     yarn run testAnalysisEngine --silent)

news/3 Code Health/1247.md

@@ -0,0 +1 @@
+Parallelize jobs (unit tests) on CI server.

src/test/debugger/common/constants.ts

@@ -4,4 +4,4 @@
 'use strict';
 
 // Sometimes PTVSD can take a while for thread & other events to be reported.
-export const DEBUGGER_TIMEOUT = 10000;
+export const DEBUGGER_TIMEOUT = 20000;

src/test/debugger/misc.test.ts

@@ -3,8 +3,7 @@
 
 // tslint:disable:no-suspicious-comment max-func-body-length no-invalid-this no-var-requires no-require-imports no-any
 
-import { expect, use } from 'chai';
-import * as chaiAsPromised from 'chai-as-promised';
+import { expect } from 'chai';
 import * as path from 'path';
 import { ThreadEvent } from 'vscode-debugadapter';
 import { DebugClient } from 'vscode-debugadapter-testsupport';
@@ -22,8 +21,6 @@ import { DebugClientEx } from './debugClient';
 
 const isProcessRunning = require('is-running') as (number) => boolean;
 
-use(chaiAsPromised);
-
 const debugFilesPath = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'debugging');
 
 const DEBUG_ADAPTER = path.join(__dirname, '..', '..', 'client', 'debugger', 'Main.js');
@@ -445,12 +442,20 @@ let testCounter = 0;
             const pauseLocation = { path: path.join(debugFilesPath, 'sample3WithEx.py'), line: 5 };
             await debugClient.assertStoppedLocation('exception', pauseLocation);
         });
-        test('Test multi-threaded debugging', async () => {
+        test('Test multi-threaded debugging', async function () {
+            if (debuggerType !== 'python') {
+                // See GitHub issue #1250
+                this.skip();
+                return;
+            }
             await Promise.all([
                 debugClient.configurationSequence(),
                 debugClient.launch(buildLauncArgs('multiThread.py', false)),
                 debugClient.waitForEvent('initialized')
             ]);
+
+            // Add a delay for debugger to start (sometimes it takes a long time for new debugger to break).
+            await sleep(3000);
             const pythonFile = path.join(debugFilesPath, 'multiThread.py');
             const breakpointLocation = { path: pythonFile, column: 1, line: 11 };
             await debugClient.setBreakpointsRequest({
@@ -459,8 +464,49 @@ let testCounter = 0;
                 source: { path: breakpointLocation.path }
             });
 
-            // hit breakpoint.
             await debugClient.assertStoppedLocation('breakpoint', breakpointLocation);
+            const threads = await debugClient.threadsRequest();
+            expect(threads.body.threads).of.lengthOf(2, 'incorrect number of threads');
+            for (const thread of threads.body.threads) {
+                expect(thread.id).to.be.lessThan(MAX_SIGNED_INT32 + 1, 'ThreadId is not an integer');
+            }
+        });
+        test('Test multi-threaded debugging', async function () {
+            this.timeout(30000);
+            await Promise.all([
+                debugClient.launch(buildLauncArgs('multiThread.py', false)),
+                debugClient.waitForEvent('initialized')
+            ]);
+
+            const pythonFile = path.join(debugFilesPath, 'multiThread.py');
+            const breakpointLocation = { path: pythonFile, column: 1, line: 11 };
+            const breakpointRequestArgs = {
+                lines: [breakpointLocation.line],
+                breakpoints: [{ line: breakpointLocation.line, column: breakpointLocation.column }],
+                source: { path: breakpointLocation.path }
+            };
+
+            function waitForStoppedEventFromTwoThreads() {
+                return new Promise((resolve, reject) => {
+                    let numberOfStops = 0;
+                    debugClient.addListener('stopped', (event: DebugProtocol.StoppedEvent) => {
+                        numberOfStops += 1;
+                        if (numberOfStops < 2) {
+                            return;
+                        }
+                        resolve(event);
+                    });
+                    setTimeout(() => reject(new Error('Timeout waiting for two threads to stop at breakpoint')), DEBUGGER_TIMEOUT);
+                });
+            }
+
+            await Promise.all([
+                debugClient.setBreakpointsRequest(breakpointRequestArgs),
+                debugClient.setExceptionBreakpointsRequest({ filters: [] }),
+                debugClient.configurationDoneRequest(),
+                waitForStoppedEventFromTwoThreads(),
+                debugClient.assertStoppedLocation('breakpoint', breakpointLocation)
+            ]);
 
             const threads = await debugClient.threadsRequest();
             expect(threads.body.threads).of.lengthOf(2, 'incorrect number of threads');

src/test/pythonFiles/debugging/multiThread.py

@@ -4,7 +4,7 @@
 
 def bar():
     time.sleep(2)
-    print("abcdef")
+    print('bar')
 
 def foo(x):
     while True: