Merge pull request #419 from kapilkd13/testing-windows

Windows Compatibility: ensuring Cwltool works on windows OS

Author: mr-c

appveyor.yml

@@ -0,0 +1,28 @@
+version: .{build}-{branch}
+
+environment:
+
+  matrix:
+    - PYTHON: "C:\\Python27"
+      PYTHON_VERSION: "2.7.x" 
+      PYTHON_ARCH: "32"
+
+    - PYTHON: "C:\\Python27-x64"
+      PYTHON_VERSION: "2.7.x" 
+      PYTHON_ARCH: "64"
+
+install:
+  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
+  - "python --version"
+  - "python -c \"import struct; print(struct.calcsize('P') * 8)\""
+  - "pip install --disable-pip-version-check --user --upgrade pip"
+  - "pip install --upgrade --no-deps --force-reinstall git+git://github.com/kapilkd13/schema_salad@windows#egg=schema_salad-2.4.201706261942"
+
+build_script:
+  - "%CMD_IN_ENV% python setup.py install"
+
+
+test_script:
+
+  - "%CMD_IN_ENV% python setup.py test"
+

cwltool/builder.py

@@ -1,5 +1,6 @@
 from __future__ import absolute_import
 import copy
+import os
 from typing import Any, Callable, Dict, List, Text, Type, Union
 
 import six
@@ -15,7 +16,7 @@
 from .pathmapper import (PathMapper, get_listing, normalizeFilesDirs,
                          visit_class)
 from .stdfsaccess import StdFsAccess
-from .utils import aslist
+from .utils import aslist, get_feature, docker_windows_path_adjust, onWindows
 
 # if six.PY3:
 # AvroSchemaFromJSONData = avro.schema.SchemaFromJSONData
@@ -185,6 +186,11 @@ def tostr(self, value):  # type: (Any) -> Text
         if isinstance(value, dict) and value.get("class") in ("File", "Directory"):
             if "path" not in value:
                 raise WorkflowException(u"%s object missing \"path\": %s" % (value["class"], value))
+
+            # Path adjust for windows file path when passing to docker, docker accepts unix like path only
+            (docker_req, docker_is_req) = get_feature(self, "DockerRequirement")
+            if onWindows() and docker_req is not None:  # docker_req is none only when there is no dockerRequirement mentioned in hints and Requirement
+                return docker_windows_path_adjust(value["path"])
             return value["path"]
         else:
             return Text(value)

cwltool/draft2tool.py

@@ -28,7 +28,7 @@
                       _logger_validation_warnings, compute_checksums,
                       normalizeFilesDirs, shortname, uniquename)
 from .stdfsaccess import StdFsAccess
-from .utils import aslist
+from .utils import aslist, docker_windows_path_adjust, convert_pathsep_to_unix
 from six.moves import map
 
 ACCEPTLIST_EN_STRICT_RE = re.compile(r"^[a-zA-Z0-9._+-]+$")
@@ -107,7 +107,7 @@ def revmap_file(builder, outdir, f):
 
     if "location" in f:
         if f["location"].startswith("file://"):
-            path = uri_file_path(f["location"])
+            path = convert_pathsep_to_unix(uri_file_path(f["location"]))
             revmap_f = builder.pathmapper.reversemap(path)
             if revmap_f:
                 f["location"] = revmap_f[1]
@@ -156,7 +156,7 @@ def run(self, **kwargs):
 def check_adjust(builder, f):
     # type: (Builder, Dict[Text, Any]) -> Dict[Text, Any]
 
-    f["path"] = builder.pathmapper.mapper(f["location"])[1]
+    f["path"] = docker_windows_path_adjust(builder.pathmapper.mapper(f["location"])[1])
     f["dirname"], f["basename"] = os.path.split(f["path"])
     if f["class"] == "File":
         f["nameroot"], f["nameext"] = os.path.splitext(f["basename"])
@@ -230,6 +230,10 @@ def job(self,
             (docker_req, docker_is_req) = self.get_requirement("DockerRequirement")
             if docker_req and kwargs.get("use_container") is not False:
                 dockerimg = docker_req.get("dockerImageId") or docker_req.get("dockerPull")
+            elif kwargs.get("default_container", None) is not None and kwargs.get("use_container") is not False:
+                dockerimg = kwargs.get("default_container")
+
+            if dockerimg:
                 cmdline = ["docker", "run", dockerimg] + cmdline
             keydict = {u"cmdline": cmdline}
 

cwltool/expression.py

@@ -4,6 +4,7 @@
 import logging
 import re
 from typing import Any, AnyStr, Dict, List, Text, Union
+from .utils import docker_windows_path_adjust
 import six
 from six import u
 
@@ -203,8 +204,8 @@ def do_eval(ex, jobinput, requirements, outdir, tmpdir, resources,
     # type: (Union[dict, AnyStr], Dict[Text, Union[Dict, List, Text]], List[Dict[Text, Any]], Text, Text, Dict[Text, Union[int, Text]], Any, bool, int, bool) -> Any
 
     runtime = copy.copy(resources)
-    runtime["tmpdir"] = tmpdir
-    runtime["outdir"] = outdir
+    runtime["tmpdir"] = docker_windows_path_adjust(tmpdir)
+    runtime["outdir"] = docker_windows_path_adjust(outdir)
 
     rootvars = {
         u"inputs": jobinput,

cwltool/job.py

@@ -16,6 +16,7 @@
 
 import shellescape
 
+from .utils import copytree_with_merge, docker_windows_path_adjust, onWindows
 from . import docker
 from .builder import Builder
 from .docker_uid import docker_vm_uid
@@ -106,8 +107,14 @@ def relink_initialworkdir(pathmapper, inplace_update=False):
             if os.path.islink(vol.target) or os.path.isfile(vol.target):
                 os.remove(vol.target)
             elif os.path.isdir(vol.target):
-                os.rmdir(vol.target)
-            os.symlink(vol.resolved, vol.target)
+                shutil.rmtree(vol.target)
+            if onWindows():
+                if vol.type in ("File", "WritableFile"):
+                    shutil.copy(vol.resolved,vol.target)
+                elif vol.type in ("Directory", "WritableDirectory"):
+                    copytree_with_merge(vol.resolved, vol.target)
+            else:
+                os.symlink(vol.resolved, vol.target)
 
 class JobBase(object):
     def __init__(self):  # type: () -> None
@@ -278,13 +285,16 @@ def run(self, pull_image=True, rm_container=True,
         if vars_to_preserve is not None:
             for key, value in os.environ.items():
                 if key in vars_to_preserve and key not in env:
-                    env[key] = value
-        env["HOME"] = self.outdir
-        env["TMPDIR"] = self.tmpdir
-
-        stageFiles(self.pathmapper, os.symlink, ignoreWritable=True)
+                    # On Windows, subprocess env can't handle unicode.
+                    env[key] = str(value) if onWindows() else value
+        env["HOME"] = str(self.outdir) if onWindows() else self.outdir
+        env["TMPDIR"] = str(self.tmpdir) if onWindows() else self.tmpdir
+        if "PATH" not in env:
+            env["PATH"] = str(os.environ["PATH"]) if onWindows() else os.environ["PATH"]
+
+        stageFiles(self.pathmapper, ignoreWritable=True, symLink=True)
         if self.generatemapper:
-            stageFiles(self.generatemapper, os.symlink, ignoreWritable=self.inplace_update)
+            stageFiles(self.generatemapper, ignoreWritable=self.inplace_update, symLink=True)
             relink_initialworkdir(self.generatemapper, inplace_update=self.inplace_update)
 
         self._execute([], env, rm_tmpdir=rm_tmpdir, move_outputs=move_outputs)
@@ -306,25 +316,26 @@ def add_volumes(self, pathmapper, runtime, stage_output):
                 containertgt = vol.target
             if vol.type in ("File", "Directory"):
                 if not vol.resolved.startswith("_:"):
-                    runtime.append(u"--volume=%s:%s:ro" % (vol.resolved, containertgt))
+                    runtime.append(u"--volume=%s:%s:ro" % (docker_windows_path_adjust(vol.resolved), docker_windows_path_adjust(containertgt)))
             elif vol.type == "WritableFile":
                 if self.inplace_update:
-                    runtime.append(u"--volume=%s:%s:rw" % (vol.resolved, containertgt))
+                    runtime.append(u"--volume=%s:%s:rw" % (docker_windows_path_adjust(vol.resolved), docker_windows_path_adjust(containertgt)))
                 else:
                     shutil.copy(vol.resolved, vol.target)
             elif vol.type == "WritableDirectory":
                 if vol.resolved.startswith("_:"):
                     os.makedirs(vol.target, 0o0755)
                 else:
                     if self.inplace_update:
-                        runtime.append(u"--volume=%s:%s:rw" % (vol.resolved, containertgt))
+                        runtime.append(u"--volume=%s:%s:rw" % (docker_windows_path_adjust(vol.resolved), docker_windows_path_adjust(containertgt)))
                     else:
                         shutil.copytree(vol.resolved, vol.target)
             elif vol.type == "CreateFile":
                 createtmp = os.path.join(host_outdir, os.path.basename(vol.target))
                 with open(createtmp, "wb") as f:
                     f.write(vol.resolved.encode("utf-8"))
-                runtime.append(u"--volume=%s:%s:ro" % (createtmp, vol.target))
+                runtime.append(u"--volume=%s:%s:ro" % (docker_windows_path_adjust(createtmp), docker_windows_path_adjust(vol.target)))
+
 
     def run(self, pull_image=True, rm_container=True,
             rm_tmpdir=True, move_outputs="move", **kwargs):
@@ -361,14 +372,14 @@ def run(self, pull_image=True, rm_container=True,
 
         runtime = [u"docker", u"run", u"-i"]
 
-        runtime.append(u"--volume=%s:%s:rw" % (os.path.realpath(self.outdir), self.builder.outdir))
-        runtime.append(u"--volume=%s:%s:rw" % (os.path.realpath(self.tmpdir), "/tmp"))
+        runtime.append(u"--volume=%s:%s:rw" % (docker_windows_path_adjust(os.path.realpath(self.outdir)), self.builder.outdir))
+        runtime.append(u"--volume=%s:%s:rw" % (docker_windows_path_adjust(os.path.realpath(self.tmpdir)), "/tmp"))
 
         self.add_volumes(self.pathmapper, runtime, False)
         if self.generatemapper:
             self.add_volumes(self.generatemapper, runtime, True)
 
-        runtime.append(u"--workdir=%s" % (self.builder.outdir))
+        runtime.append(u"--workdir=%s" % (docker_windows_path_adjust(self.builder.outdir)))
         runtime.append(u"--read-only=true")
 
         if kwargs.get("custom_net", None) is not None:
@@ -379,9 +390,12 @@ def run(self, pull_image=True, rm_container=True,
         if self.stdout:
             runtime.append("--log-driver=none")
 
-        euid = docker_vm_uid() or os.geteuid()
+        if onWindows():  # windows os dont have getuid or geteuid functions
+            euid = docker_vm_uid()
+        else:
+            euid = docker_vm_uid() or os.geteuid()
 
-        if kwargs.get("no_match_user", None) is False:
+        if kwargs.get("no_match_user", None) is False and euid is not None:
             runtime.append(u"--user=%s" % (euid))
 
         if rm_container:
@@ -436,7 +450,7 @@ def _job_popen(
 
         sp = subprocess.Popen(commands,
                               shell=False,
-                              close_fds=True,
+                              close_fds=not onWindows(),
                               stdin=stdin,
                               stdout=stdout,
                               stderr=stderr,
@@ -478,14 +492,14 @@ def _job_popen(
             stderr_path=stderr_path,
             stdin_path=stdin_path,
         )
-        with open(os.path.join(job_dir, "job.json"), "w") as f:
+        with open(os.path.join(job_dir, "job.json"), "wb") as f:
             json.dump(job_description, f)
         try:
             job_script = os.path.join(job_dir, "run_job.bash")
             with open(job_script, "wb") as f:
                 f.write(job_script_contents.encode('utf-8'))
             job_run = os.path.join(job_dir, "run_job.py")
-            with open(job_run, "w") as f:
+            with open(job_run, "wb") as f:
                 f.write(PYTHON_RUN_SCRIPT)
             sp = subprocess.Popen(
                 ["bash", job_script.encode("utf-8")],

cwltool/load_tool.py

@@ -47,7 +47,8 @@ def fetch_document(argsworkflow,  # type: Union[Text, Dict[Text, Any]]
     workflowobj = None  # type: CommentedMap
     if isinstance(argsworkflow, string_types):
         split = urllib.parse.urlsplit(argsworkflow)
-        if split.scheme:
+        # In case of Windows path, urlsplit misjudge Drive letters as scheme, here we are skipping that
+        if split.scheme and split.scheme in [u'http',u'https',u'file']:
             uri = argsworkflow
         elif os.path.exists(os.path.abspath(argsworkflow)):
             uri = file_uri(str(os.path.abspath(argsworkflow)))

cwltool/main.py

@@ -39,6 +39,8 @@
 from .software_requirements import DependenciesConfiguration, get_container_from_software_requirements, SOFTWARE_REQUIREMENTS_ENABLED
 from .stdfsaccess import StdFsAccess
 from .update import ALLUPDATES, UPDATES
+from .utils import onWindows
+from ruamel.yaml.comments import Comment, CommentedSeq, CommentedMap
 
 
 _logger = logging.getLogger("cwltool")
@@ -695,6 +697,10 @@ def main(argsl=None,  # type: List[str]
                 argsl = sys.argv[1:]
             args = arg_parser().parse_args(argsl)
 
+        # If On windows platform, A default Docker Container is Used if not explicitely provided by user
+        if onWindows() and not args.default_container:
+            args.default_container = "ubuntu"
+
         # If caller provided custom arguments, it may be not every expected
         # option is set, so fill in no-op defaults to avoid crashing when
         # dereferencing them in args.

cwltool/pathmapper.py

@@ -12,6 +12,8 @@
 from schema_salad.sourceline import SourceLine
 from six.moves import urllib
 
+from .utils import convert_pathsep_to_unix
+
 from .stdfsaccess import StdFsAccess, abspath
 
 _logger = logging.getLogger("cwltool")
@@ -186,7 +188,8 @@ def visitlisting(self, listing, stagedir, basedir, copy=False, staged=False):
 
     def visit(self, obj, stagedir, basedir, copy=False, staged=False):
         # type: (Dict[Text, Any], Text, Text, bool, bool) -> None
-        tgt = os.path.join(stagedir, obj["basename"])
+        tgt = convert_pathsep_to_unix(
+            os.path.join(stagedir, obj["basename"]))
         if obj["location"] in self._pathmap:
             return
         if obj["class"] == "Directory":