CI - better coverage report, pypy on Azure, add codeclimate (#1031)

This now is done. It's a major rewrite and was significantly more troublesome than expected. After all the effort put in, failed to convince either codecov or coveralls to function correctly. So where two does not work, let's use the third option: codeclimate (https://codeclimate.com/github/tox-dev/tox/builds). 

This one finally gives up on trying to be too smart (e.g. forcing users to merge reports locally during the build), but work correctly what it does offer:
- great documentation,
- Github App,
- great chrome extension to annotate PRs inline,
- Github comment support.

Does not have branch coverage for now but I don't think we used that much often that feature. With this, we also moved off Travis completely as now ``pypy`` tests are also done on Azure Pipelines.

PS. NOTE: we no longer really on a service to report/display code coverage. The CI itself collects all coverage reports, merges them and then displays them as a final step. Codeclimate is just an additional UI to this data, and the coverage gets uploaded to that service too; however, it's also available as part of the CI build log (both full coverage and diff coverage).

Author: gaborbernat

.codeclimate.yml

@@ -0,0 +1,20 @@
+engines:
+  duplication:
+    enabled: true
+    config:
+      languages:
+        python:
+          python_version: 3
+  pep8:
+    enabled: true
+  radon:
+    enabled: true
+  sonar-python:
+    enabled: true
+    config:
+      tests_patterns:
+      - tests/**
+
+ratings:
+  paths:
+  - "**.py"

.codecov.yml

@@ -7,13 +7,10 @@
 .. image:: https://dev.azure.com/toxdev/tox/_apis/build/status/tox%20ci?branchName=master
   :target: https://dev.azure.com/toxdev/tox/_build/latest?definitionId=9&branchName=master
   :alt: Azure Pipelines build status
-.. image:: https://travis-ci.org/tox-dev/tox.svg?branch=master
-  :target: https://travis-ci.org/tox-dev/tox
-  :alt: Travis-CI build status
-.. image:: https://codecov.io/gh/tox-dev/tox/branch/master/graph/badge.svg
-  :target: https://codecov.io/gh/tox-dev/tox
-  :alt: Code coverage Status
-.. image:: https://readthedocs.org/projects/tox/badge/?version=latest
+.. image:: https://api.codeclimate.com/v1/badges/425c19ab2169a35e1c16/test_coverage
+   :target: https://codeclimate.com/github/tox-dev/tox/test_coverage
+   :alt: Test Coverage
+.. image:: https://readthedocs.org/projects/tox/badge/?version=latest&style=flat-square
   :target: https://tox.readthedocs.io/en/latest/?badge=latest
   :alt: Documentation status
 .. image:: https://img.shields.io/badge/code%20style-black-000000.svg

.travis.yml

@@ -2,6 +2,11 @@ name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr)
 
 variables:
   "System.PreferGit": true
+  CI_NAME: Azure Pipelines
+  CI_BUILD_ID: $(Build.BuildId)
+  CI_BUILD_URL: "https://toxdev.visualstudio.com/tox/_build/results?buildId=$(Build.BuildId)"
+  GIT_BRANCH: $(Build.SourceBranch)
+  GIT_COMMIT_SHA: $(Build.SourceVersion)
 
 trigger:
   batch: true
@@ -23,63 +28,154 @@ trigger:
     - tasks/*
 
 jobs:
+- job: notify_build_start
+  pool: {vmImage: 'Ubuntu 16.04'}
+  steps:
+  - script: echo start
+  - script: |
+      curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter && \
+      chmod +x ./cc-test-reporter && \
+      ./cc-test-reporter before-build -d
+    displayName: notify code climate of new build
+    condition: eq(variables['system.pullrequest.isfork'], false)
+
+- template: azure-run-tox-env.yml
+  parameters: {tox: fix_lint, python: 3.7}
+- template: azure-run-tox-env.yml
+  parameters: {tox: docs, python: 3.7}
 - template: azure-run-tox-env.yml
-  parameters: {name: check_code_style, tox: "fix-lint", python: '3.7',  image: 'Ubuntu 16.04'}
+  parameters: {tox: package_description, python: 3.7}
+
 - template: azure-run-tox-env.yml
-  parameters: {name: generate_docs, tox: docs, python: '3.7', image: 'Ubuntu 16.04'}
+  parameters: {tox: pypy, python: pypy, os: linux}
 - template: azure-run-tox-env.yml
-  parameters: {name: check_package_long_description, tox: "package-description", python: '3.7',  image: 'Ubuntu 16.04'}
+  parameters: {tox: pypy3, python: pypy3, os: linux}
 
 - template: azure-run-tox-env.yml
-  parameters: {name: windows_37, python: '3.7',  image: 'vs2017-win2016'}
+  parameters: {tox: py37, python: 3.7, os: windows}
 - template: azure-run-tox-env.yml
-  parameters: {name: windows_36, python: '3.6',  image: 'vs2017-win2016'}
+  parameters: {tox: py36, python: 3.6, os: windows}
 - template: azure-run-tox-env.yml
-  parameters: {name: windows_35, python: '3.5',  image: 'vs2017-win2016'}
+  parameters: {tox: py35, python: 3.5, os: windows}
 - template: azure-run-tox-env.yml
-  parameters: {name: windows_34, python: '3.4',  image: 'vs2017-win2016'}
+  parameters: {tox: py34, python: 3.4, os: windows}
 - template: azure-run-tox-env.yml
-  parameters: {name: windows_27, python: '2.7',  image: 'vs2017-win2016'}
+  parameters: {tox: py27, python: 2.7, os: windows}
 
 - template: azure-run-tox-env.yml
-  parameters: {name: linux_37, python: '3.7',  image: 'Ubuntu 16.04'}
+  parameters: {tox: py37, python: 3.7, os: linux}
 - template: azure-run-tox-env.yml
-  parameters: {name: linux_36, python: '3.6',  image: 'Ubuntu 16.04'}
+  parameters: {tox: py36, python: 3.6, os: linux}
 - template: azure-run-tox-env.yml
-  parameters: {name: linux_35, python: '3.5',  image: 'Ubuntu 16.04'}
+  parameters: {tox: py35, python: 3.5, os: linux}
 - template: azure-run-tox-env.yml
-  parameters: {name: linux_34, python: '3.4',  image: 'Ubuntu 16.04'}
+  parameters: {tox: py34, python: 3.4, os: linux}
 - template: azure-run-tox-env.yml
-  parameters: {name: linux_27, python: '2.7',  image: 'Ubuntu 16.04'}
+  parameters: {tox: py27, python: 2.7, os: linux}
 
 - template: azure-run-tox-env.yml
-  parameters: {name: macOS_36, python: '3.6',  image: 'macOS 10.13'}
+  parameters: {tox: py36, python: 3.6, os: macOs}
 
-- job: tests_done
+- job: report_coverage
   pool: {vmImage: 'Ubuntu 16.04'}
-  condition: always()
+  condition: eq(variables['system.pullrequest.isfork'], false)
   dependsOn:
-  - windows_37
-  - windows_36
-  - windows_35
-  - windows_34
-  - windows_27
-  - linux_37
-  - linux_36
-  - linux_35
-  - linux_34
-  - linux_27
-  - macOS_36
+  - windows_py37
+  - windows_py36
+  - windows_py35
+  - windows_py34
+  - windows_py27
+  - linux_py37
+  - linux_py36
+  - linux_py35
+  - linux_py34
+  - linux_py27
+  - linux_pypy3
+  - linux_pypy
+  - macOS_py36
   steps:
-  - script: echo "done"
-    displayName: running tests done
+  - task: DownloadBuildArtifacts@0
+    displayName: download coverage files for run
+    inputs:
+      buildType: current
+      downloadType: specific
+      itemPattern: coverage-*/*
+      downloadPath: $(Build.StagingDirectory)
+
+  - task: UsePythonVersion@0
+    displayName: setup python
+    inputs:
+      versionSpec: 3.7
+
+  - script: |
+      python -c '
+      from pathlib import Path
+      import shutil
+
+      from_folder = Path("$(Build.StagingDirectory)")
+      destination_folder = Path("$(System.DefaultWorkingDirectory)") / ".tox"
+      destination_folder.mkdir()
+      for coverage_file in from_folder.glob("*/.coverage"):
+          destination = destination_folder / f".coverage.{coverage_file.parent.name[9:]}"
+          print(f"{coverage_file} copy to {destination}")
+          shutil.copy(str(coverage_file), str(destination))'
+    displayName: move coverage files into .tox
+
+  - script: 'python -m pip install -U tox --pre'
+    displayName: install tox
+
+  - script: 'python -m tox -e py --sdistonly'
+    displayName: generate version.py
+
+  - script: 'python -m tox -e coverage'
+    displayName: create coverag report via tox
+
+  - task: PublishCodeCoverageResults@1
+    displayName: publish overall coverage report to Azure
+    inputs:
+      codeCoverageTool: 'cobertura'
+      summaryFileLocation: '$(System.DefaultWorkingDirectory)/.tox/coverage.xml'
+      reportDirectory: '$(System.DefaultWorkingDirectory)/.tox/htmlcov'
+      failIfCoverageEmpty: true
+
+  - script: |
+      curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter && \
+      chmod +x ./cc-test-reporter && \
+      python -c '
+      from xml.etree import ElementTree as et
+      from pathlib import Path
+      import subprocess
+
+      from_folder = Path("$(Build.StagingDirectory)")
+      for counter, coverage_file in enumerate(from_folder.glob("*/coverage.xml")):
+          key = coverage_file.parent.name[9:]
+          print(f"{counter}) {coverage_file}")
+          try:
+              cmd = ["$(System.DefaultWorkingDirectory)/cc-test-reporter", "format-coverage",
+                     str(coverage_file),
+                     "-d", "-t", "coverage.py",
+                     "-o", f"$(Build.StagingDirectory)/code-climate.{key}.json"]
+              print(f"\t{cmd}")
+              log = subprocess.check_output(cmd, stderr=subprocess.STDOUT, universal_newlines=True)
+              code = 0
+          except subprocess.CalledProcessError as exception:
+              log, code = exception.output, exception.returncode
+          finally:
+              print(code, log, "\n", sep="\n")' && \
+      ./cc-test-reporter sum-coverage -d --output - \
+          --parts $(ls -1 $(Build.StagingDirectory)/code-climate.*.json | wc -l) \
+          $(Build.StagingDirectory)/code-climate.*.json | \
+      ./cc-test-reporter -d -r d24f105984ab5e087773a21b8668acb0b36cb8311fc2637f78a2d9451e531e08 \
+          upload-coverage --input -
+    displayName: publish code climate
+    condition: succeededOrFailed()
 
 - job: publish
   dependsOn:
-  - tests_done
-  - check_code_style
-  - generate_docs
-  - check_package_long_description
+  - report_coverage
+  - linux_fix_lint
+  - linux_docs
+  - linux_package_description
   condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'))
   pool: {vmImage: 'Ubuntu 16.04'}
   steps:
@@ -90,5 +186,5 @@ jobs:
     displayName: Package and publish to PyPI
     inputs:
       pypiConnection: pypi-conn
-      packageDirectory: '$(System.DefaultWorkingDirectory)'
+      packageDirectory: $(System.DefaultWorkingDirectory)
       alsoPublishWheel: true