Merge pull request #742 from common-workflow-language/windows-Jenkinsfile

Better windows support

Author: mr-c

Jenkinsfile

@@ -0,0 +1,26 @@
+pipeline {
+  agent {
+    node {
+      label 'windows'
+    }
+
+  }
+  stages {
+    stage('build') {
+      steps {
+        withPythonEnv(pythonInstallation: 'Windows-CPython-36') {
+          pybat(script: 'pip install .', returnStdout: true)
+          pybat 'jenkins.bat'
+        }
+
+      }
+    }
+  }
+  post {
+    always {
+      junit 'tests.xml'
+
+    }
+
+  }
+}

Makefile

@@ -114,7 +114,7 @@ pylint_report.txt: ${PYSOURCES}
 diff_pylint_report: pylint_report.txt
 	diff-quality --violations=pylint pylint_report.txt
 
-.coverage: tests
+.coverage: testcov
 
 coverage: .coverage
 	coverage report
@@ -135,7 +135,11 @@ diff-cover.html: coverage-gcovr.xml coverage.xml
 		--html-report diff-cover.html
 
 ## test        : run the ${MODULE} test suite
-test: $(PYSOURCES)
+test: $(pysources)
+	python setup.py test
+
+## testcov     : run the ${MODULE} test suite and collect coverage
+testcov: $(pysources)
 	python setup.py test --addopts "--cov cwltool"
 
 sloccount.sc: ${PYSOURCES} Makefile

appveyor.yml

@@ -46,9 +46,13 @@ install:
 build_script:
   - "%PYTHON%\\python.exe -m pip install ."
 
-
 test_script:
-  - "%PYTHON%\\python.exe -m pytest --verbose -p no:cacheprovider"
+  - "%PYTHON%\\python.exe -m pytest --verbose -p no:cacheprovider --junit-xml=tests.xml"
+
+after_test:
+  - ps: |
+      $wc = New-Object 'System.Net.WebClient'
+      $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($Env:APPVEYOR_JOB_ID)", (Resolve-Path .\tests.xml))
 
 branches:
   only:

cwltool/docker.py

@@ -28,6 +28,41 @@
 
 found_images = set()  # type: Set[Text]
 found_images_lock = threading.Lock()
+__docker_machine_mounts = None  # type: List[Text]
+__docker_machine_mounts_lock = threading.Lock()
+
+def _get_docker_machine_mounts():  # type: () -> List[Text]
+    global __docker_machine_mounts
+    if __docker_machine_mounts is None:
+        with __docker_machine_mounts_lock:
+            if 'DOCKER_MACHINE_NAME' not in os.environ:
+                __docker_machine_mounts = []
+            else:
+                __docker_machine_mounts = [u'/' + line.split(None, 1)[0]
+                        for line in subprocess.check_output(
+                            ['docker-machine', 'ssh',
+                                os.environ['DOCKER_MACHINE_NAME'],
+                                'mount', '-t', 'vboxsf'],
+                            universal_newlines=True).splitlines()]
+    return __docker_machine_mounts
+
+def _check_docker_machine_path(path):  # type: (Text) -> None
+    mounts = _get_docker_machine_mounts()
+    if mounts:
+        found = False
+        for mount in mounts:
+            if path.startswith(mount):
+                found = True
+                break
+        if not found:
+            raise WorkflowException(
+                "Input path {path} is not in the list of host paths mounted "
+                "into the Docker virtual machine named {name}. Already mounted "
+                "paths: {mounts}.\n"
+                "See https://docs.docker.com/toolbox/toolbox_install_windows/#optional-add-shared-directories"
+                " for instructions on how to add this path to your VM.".format(path=path,
+                    name=os.environ["DOCKER_MACHINE_NAME"], mounts=mounts))
+
 
 class DockerCommandLineJob(ContainerCommandLineJob):
 
@@ -162,6 +197,8 @@ def add_volumes(self, pathmapper, runtime, secret_store=None):
                 host_outdir_tgt = None
             if vol.type in ("File", "Directory"):
                 if not vol.resolved.startswith("_:"):
+                    _check_docker_machine_path(docker_windows_path_adjust(
+                        vol.resolved))
                     runtime.append(u"--volume=%s:%s:ro" % (
                         docker_windows_path_adjust(vol.resolved),
                         docker_windows_path_adjust(vol.target)))

cwltool/factory.py

@@ -38,25 +38,28 @@ def __init__(self,
                  makeTool=workflow.defaultMakeTool,  # type: tCallable[[Any], Process]
                  # should be tCallable[[Dict[Text, Any], Any], Process] ?
                  executor=None,  # type: tCallable[...,Tuple[Dict[Text,Any], Text]]
-                 **execkwargs  # type: Any
+                 makekwargs={},  # type: Dict[Any, Any]
+                 **execkwargs    # type: Dict[Any, Any]
                  ):
         # type: (...) -> None
         self.makeTool = makeTool
         if executor is None:
             executor = SingleJobExecutor()
         self.executor = executor
 
-        kwargs = get_default_args()
-        kwargs.pop("job_order")
-        kwargs.pop("workflow")
-        kwargs.pop("outdir")
-        kwargs.update(execkwargs)
-        self.execkwargs = kwargs
+        newExecKwargs = get_default_args()
+        newExecKwargs.pop("job_order")
+        newExecKwargs.pop("workflow")
+        newExecKwargs.pop("outdir")
+        newExecKwargs.update(execkwargs)
+        self.execkwargs = newExecKwargs
+        self.makekwargs = makekwargs
 
     def make(self, cwl):
         """Instantiate a CWL object from a CWl document."""
         load = load_tool.load_tool(cwl, self.makeTool,
-                                   strict=self.execkwargs.get("strict", True))
+                                   strict=self.execkwargs.get("strict", True),
+                                   kwargs=self.makekwargs)
         if isinstance(load, int):
             raise Exception("Error loading tool")
         return Callable(load, self)

jenkins.bat

@@ -0,0 +1,8 @@
+set PATH=%PATH%;"C:\\Program Files\\Docker Toolbox\\"
+docker-machine start default
+REM Set the environment variables to use docker-machine and docker commands
+FOR /f "tokens=*" %%i IN ('docker-machine env --shell cmd default') DO %%i
+
+python setup.py test --addopts "--junit-xml=tests.xml --cov-report xml --cov cwltool"
+pip install codecov
+codecov

tests/test_examples.py

@@ -30,7 +30,8 @@
 from cwltool.main import main
 from cwltool.utils import onWindows
 
-from .util import get_data, needs_docker
+from .util import (get_data, needs_docker, get_windows_safe_factory,
+        windows_needs_docker)
 
 sys.argv = ['']
 
@@ -135,10 +136,11 @@ def test_params(self):
         self.assertEqual(expr.interpolate("$(foo[\"b\\'ar\"].baz) $(foo[\"b\\'ar\"].baz)", inputs), "true true")
         self.assertEqual(expr.interpolate("$(foo['b\\\"ar'].baz) $(foo['b\\\"ar'].baz)", inputs), "null null")
 
-
 class TestFactory(unittest.TestCase):
+
+    @windows_needs_docker
     def test_factory(self):
-        f = cwltool.factory.Factory()
+        f = get_windows_safe_factory()
         echo = f.make(get_data("tests/echo.cwl"))
         self.assertEqual(echo(inp="foo"), {"out": "foo\n"})
 

tests/test_fetch.py

@@ -12,6 +12,7 @@
 from cwltool.main import main
 from cwltool.workflow import defaultMakeTool
 from cwltool.resolver import resolve_local
+from cwltool.utils import onWindows
 
 if sys.version_info < (3, 4):
     from pathlib2 import Path
@@ -71,15 +72,25 @@ class ResolverTest(unittest.TestCase):
     def test_resolve_local(self):
         origpath = os.getcwd()
         os.chdir(os.path.join(get_data("")))
+        def norm(uri):
+            if onWindows():
+                return uri.lower()
+            else:
+                return uri
         try:
             root = Path.cwd()
             rooturi = root.as_uri()
-            self.assertEqual(rooturi+"/tests/echo.cwl", resolve_local(None, os.path.join("tests", "echo.cwl")))
-            self.assertEqual(rooturi+"/tests/echo.cwl#main", resolve_local(None, os.path.join("tests", "echo.cwl")+"#main"))
-            self.assertEqual(rooturi+"/tests/echo.cwl", resolve_local(None, str(root / "tests" / "echo.cwl")))
-            # On Windows and Python 2.7, the left side of this test returns
-            # file:///C:/ (uppercase drive letter) and the right side returns
-            # file:///c:/ (lowercase drive letter) so force a lowercase comparison.
-            self.assertEqual((rooturi+"/tests/echo.cwl#main").lower(), resolve_local(None, str(root / "tests" / "echo.cwl")+"#main").lower())
+            self.assertEqual(norm(rooturi+"/tests/echo.cwl"),
+                    norm(resolve_local(None, os.path.join("tests",
+                        "echo.cwl"))))
+            self.assertEqual(norm(rooturi+"/tests/echo.cwl#main"),
+                    norm(resolve_local(None, os.path.join("tests",
+                        "echo.cwl")+"#main")))
+            self.assertEqual(norm(rooturi+"/tests/echo.cwl"),
+                    norm(resolve_local(None, str(root / "tests" /
+                        "echo.cwl"))))
+            self.assertEqual(norm(rooturi+"/tests/echo.cwl#main"),
+                    norm(resolve_local(None, str(root / "tests" /
+                        "echo.cwl")+"#main")))
         finally:
             os.chdir(origpath)

tests/test_iwdr.py

@@ -2,15 +2,16 @@
 
 import cwltool
 import cwltool.factory
-from .util import get_data
+from .util import get_data, get_windows_safe_factory, windows_needs_docker
 
 
 class TestInitialWorkDir(unittest.TestCase):
 
+    @windows_needs_docker
     def test_newline_in_entry(self):
         """
         test that files in InitialWorkingDirectory are created with a newline character
         """
-        f = cwltool.factory.Factory()
+        f = get_windows_safe_factory()
         echo = f.make(get_data("tests/wf/iwdr-entry.cwl"))
         self.assertEqual(echo(message="hello"), {"out": "CONFIGVAR=hello\n"})

tests/test_js_sandbox.py

@@ -13,7 +13,7 @@
                                exec_js_process)
 import cwltool.sandboxjs
 from cwltool.utils import onWindows
-from .util import get_data
+from .util import get_data, get_windows_safe_factory, windows_needs_docker
 import pytest
 
 
@@ -47,8 +47,9 @@ def test_is_javascript_installed(self):
 
 class TestValueFrom(unittest.TestCase):
 
+    @windows_needs_docker
     def test_value_from_two_concatenated_expressions(self):
-        f = cwltool.factory.Factory()
+        f = get_windows_safe_factory()
         echo = f.make(get_data("tests/wf/vf-concat.cwl"))
         self.assertEqual(echo(), {u"out": u"a sting\n"})
 

tests/test_parallel.py

@@ -7,27 +7,26 @@
 import cwltool.factory
 from cwltool.executors import MultithreadedJobExecutor
 from cwltool.utils import onWindows
-from .util import get_data
+from .util import get_data, get_windows_safe_factory, windows_needs_docker
 
 
 class TestParallel(unittest.TestCase):
-    @pytest.mark.skipif(onWindows(),
-                        reason="Unexplainable behavior: cwltool on AppVeyor does not recognize cwlVersion"
-                               "in count-lines1-wf.cwl")
+    @windows_needs_docker
     def test_sequential_workflow(self):
         test_file = "tests/wf/count-lines1-wf.cwl"
-        f = cwltool.factory.Factory(executor=MultithreadedJobExecutor())
+        f = get_windows_safe_factory(executor=MultithreadedJobExecutor())
         echo = f.make(get_data(test_file))
         self.assertEqual(echo(file1= {
                 "class": "File",
                 "location": get_data("tests/wf/whale.txt")
             }),
             {"count_output": 16})
 
+    @windows_needs_docker
     def test_scattered_workflow(self):
         test_file = "tests/wf/scatter-wf4.cwl"
         job_file = "tests/wf/scatter-job2.json"
-        f = cwltool.factory.Factory(executor=MultithreadedJobExecutor())
+        f = get_windows_safe_factory(executor=MultithreadedJobExecutor())
         echo = f.make(get_data(test_file))
         with open(get_data(job_file)) as job:
             self.assertEqual(echo(**json.load(job)), {'out': ['foo one three', 'foo two four']})

tests/test_toolargparse.py

@@ -73,9 +73,6 @@ def test_help(self):
             f.close()
             self.assertEquals(main(["--debug", f.name, '--input',
                 get_data('tests/echo.cwl')]), 0)
-            self.assertEquals(main(["--debug", f.name, '--input',
-                get_data('tests/echo.cwl')]), 0)
-
 
     @needs_docker
     def test_bool(self):

tests/util.py

@@ -1,11 +1,28 @@
 from __future__ import absolute_import
 import os
+import functools
 
 from pkg_resources import (Requirement, ResolutionError,  # type: ignore
                            resource_filename)
 import distutils.spawn
 import pytest
 
+from cwltool.utils import onWindows, windows_default_container_id
+from cwltool.factory import Factory
+
+def get_windows_safe_factory(**execkwargs):
+    if onWindows():
+        makekwargs = {'find_default_container': functools.partial(
+            force_default_container, windows_default_container_id),
+                      'use_container': True}
+        execkwargs['default_container'] = windows_default_container_id
+    else:
+        makekwargs = {}
+    return Factory(makekwargs=makekwargs, **execkwargs)
+
+def force_default_container(default_container_id, builder):
+   return default_container_id
+
 def get_data(filename):
     filename = os.path.normpath(
         filename)  # normalizing path depending on OS or else it will cause problem when joining path
@@ -26,3 +43,8 @@ def get_data(filename):
 needs_docker = pytest.mark.skipif(not bool(distutils.spawn.find_executable('docker')),
                                   reason="Requires the docker executable on the "
                                   "system path.")
+
+windows_needs_docker = pytest.mark.skipif(
+        onWindows() and not bool(distutils.spawn.find_executable('docker')),
+        reason="Running this test on MS Windows requires the docker executable "
+        "on the system path.")