PHPLIB-582: Create JSON test result file for evergreen

.evergreen/config.yml

@@ -253,13 +253,6 @@ functions:
             chmod +x $i
           done
 
-  "init test-results":
-    - command: shell.exec
-      params:
-        script: |
-          ${PREPARE_SHELL}
-          echo '{"results": [{ "status": "FAIL", "test_file": "Build", "log_raw": "No test-results.json found was created"  } ]}' > ${PROJECT_DIRECTORY}/test-results.json
-
   "install dependencies":
     - command: shell.exec
       params:
@@ -275,7 +268,6 @@ pre:
   - func: "prepare resources"
   - func: "windows fix"
   - func: "fix absolute paths"
-  - func: "init test-results"
   - func: "make files executable"
   - func: "install dependencies"
 

.evergreen/run-tests.sh

@@ -26,10 +26,10 @@ PATH=/opt/php/${PHP_VERSION}-64bit/bin:$OLD_PATH
 case "$TESTS" in
    atlas-data-lake*)
       MONGODB_URI="mongodb://mhuser:[email protected]:27017"
-      php vendor/bin/phpunit --testsuite "Atlas Data Lake Test Suite"
+      php vendor/bin/phpunit --configuration phpunit.evergreen.xml --testsuite "Atlas Data Lake Test Suite"
       ;;
 
    *)
-      php vendor/bin/phpunit
+      php vendor/bin/phpunit --configuration phpunit.evergreen.xml
       ;;
 esac

composer.json

@@ -16,7 +16,7 @@
         "jean85/pretty-package-versions": "^1.2"
     },
     "require-dev": {
-        "phpunit/phpunit": "^6.4 || ^8.3",
+        "phpunit/phpunit": "^6.4",
         "sebastian/comparator": "^2.0 || ^3.0",
         "squizlabs/php_codesniffer": "^3.5, <3.5.5",
         "symfony/phpunit-bridge": "^4.4@dev"

phpunit.evergreen.xml

@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<phpunit
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:noNamespaceSchemaLocation="https://schema.phpunit.de/8.3/phpunit.xsd"
+         beStrictAboutOutputDuringTests="true"
+         beStrictAboutChangesToGlobalState="true"
+         colors="true"
+         bootstrap="tests/bootstrap.php"
+         defaultTestSuite="Default Test Suite"
+>
+
+    <php>
+        <ini name="error_reporting" value="-1"/>
+        <env name="MONGODB_URI" value="mongodb://127.0.0.1:27017/?serverSelectionTimeoutMS=100"/>
+        <env name="MONGODB_DATABASE" value="phplib_test"/>
+    </php>
+
+    <testsuites>
+        <testsuite name="Default Test Suite">
+            <directory>./tests/</directory>
+        </testsuite>
+
+        <testsuite name="Atlas Data Lake Test Suite">
+            <file>tests/SpecTests/AtlasDataLakeSpecTest.php</file>
+        </testsuite>
+    </testsuites>
+
+    <listeners>
+        <listener class="MongoDB\Tests\EvergreenLogListener">
+            <arguments>
+                <string>test-results.json</string>
+            </arguments>
+        </listener>
+    </listeners>
+</phpunit>

tests/EvergreenLogListener.php

@@ -0,0 +1,205 @@
+<?php
+
+namespace MongoDB\Tests;
+
+use Exception;
+use PHPUnit\Framework\AssertionFailedError;
+use PHPUnit\Framework\SelfDescribing;
+use PHPUnit\Framework\Test;
+use PHPUnit\Framework\TestFailure;
+use PHPUnit\Framework\TestListener;
+use PHPUnit\Framework\TestSuite;
+use PHPUnit\Framework\Warning;
+use PHPUnit\Util\Filter;
+use PHPUnit\Util\Printer;
+use function get_class;
+use function json_encode;
+use function round;
+
+// phpcs:disable SlevomatCodingStandard.Exceptions.ReferenceThrowableOnly.ReferencedGeneralException
+class EvergreenLogListener extends Printer implements TestListener
+{
+    /** @var array[] */
+    private $tests = [];
+
+    /** @var array|null */
+    private $currentTestCase = null;
+
+    /**
+     * Flush buffer and close output.
+     */
+    public function flush()
+    {
+        $this->write($this->getJson());
+
+        parent::flush();
+    }
+
+    /**
+     * An error occurred.
+     *
+     * @param Test      $test
+     * @param Exception $e
+     * @param float     $time
+     */
+    public function addError(Test $test, Exception $e, $time)
+    {
+        $this->doAddFault($test, $e, 'fail');
+    }
+
+    /**
+     * A warning occurred.
+     *
+     * @param Test    $test
+     * @param Warning $e
+     * @param float   $time
+     */
+    public function addWarning(Test $test, Warning $e, $time)
+    {
+        // Do nothing for now
+    }
+
+    /**
+     * A failure occurred.
+     *
+     * @param Test                 $test
+     * @param AssertionFailedError $e
+     * @param float                $time
+     */
+    public function addFailure(Test $test, AssertionFailedError $e, $time)
+    {
+        $this->doAddFault($test, $e, 'fail');
+    }
+
+    /**
+     * Incomplete test.
+     *
+     * @param Test      $test
+     * @param Exception $e
+     * @param float     $time
+     */
+    public function addIncompleteTest(Test $test, Exception $e, $time)
+    {
+        $this->doAddSkipped($test);
+    }
+
+    /**
+     * Risky test.
+     *
+     * @param Test      $test
+     * @param Exception $e
+     * @param float     $time
+     */
+    public function addRiskyTest(Test $test, Exception $e, $time)
+    {
+        return;
+    }
+
+    /**
+     * Skipped test.
+     *
+     * @param Test      $test
+     * @param Exception $e
+     * @param float     $time
+     */
+    public function addSkippedTest(Test $test, Exception $e, $time)
+    {
+        $this->doAddSkipped($test);
+    }
+
+    /**
+     * A testsuite started.
+     *
+     * @param TestSuite $suite
+     */
+    public function startTestSuite(TestSuite $suite)
+    {
+    }
+
+    /**
+     * A testsuite ended.
+     *
+     * @param TestSuite $suite
+     */
+    public function endTestSuite(TestSuite $suite)
+    {
+    }
+
+    /**
+     * A test started.
+     *
+     * @param Test $test
+     */
+    public function startTest(Test $test)
+    {
+        $this->currentTestCase = [
+            'status' => 'pass',
+            'test_file' => get_class($test) . '::' . $test->getName(),
+        ];
+    }
+
+    /**
+     * A test ended.
+     *
+     * @param Test  $test
+     * @param float $time
+     */
+    public function endTest(Test $test, $time)
+    {
+        if (! $this->currentTestCase) {
+            return;
+        }
+
+        $this->currentTestCase['elapsed'] = round($time, 6);
+
+        $this->tests[] = $this->currentTestCase;
+
+        $this->currentTestCase = null;
+    }
+
+    /**
+     * Returns the XML as a string.
+     *
+     * @return string
+     */
+    public function getJson()
+    {
+        return json_encode(['results' => $this->tests]);
+    }
+
+    /**
+     * Method which generalizes addError() and addFailure()
+     *
+     * @param Test      $test
+     * @param Exception $e
+     * @param string    $type
+     */
+    private function doAddFault(Test $test, Exception $e, $type)
+    {
+        if ($this->currentTestCase === null) {
+            return;
+        }
+
+        if ($test instanceof SelfDescribing) {
+            $buffer = $test->toString() . "\n";
+        } else {
+            $buffer = '';
+        }
+
+        $buffer .= TestFailure::exceptionToString($e) . "\n" .
+            Filter::getFilteredStacktrace($e);
+
+        $this->currentTestCase['status'] = $type;
+        $this->currentTestCase['raw_log'] = $buffer;
+    }
+
+    private function doAddSkipped(Test $test)
+    {
+        if ($this->currentTestCase === null) {
+            return;
+        }
+
+        $this->currentTestCase['status'] = 'skip';
+    }
+}
+// phpcs:enable SlevomatCodingStandard.Exceptions.ReferenceThrowableOnly.ReferencedGeneralException