Merge branch 'master' into windowsdoc

Author: mr-c

appveyor.yml

@@ -26,3 +26,6 @@ test_script:
 
   - "%CMD_IN_ENV% python setup.py test"
 
+branches:
+  only:
+    - master

cwltool/builder.py

@@ -18,9 +18,6 @@
 from .stdfsaccess import StdFsAccess
 from .utils import aslist, get_feature, docker_windows_path_adjust, onWindows
 
-# if six.PY3:
-# AvroSchemaFromJSONData = avro.schema.SchemaFromJSONData
-# else:
 AvroSchemaFromJSONData = avro.schema.make_avsc_object
 
 CONTENT_LIMIT = 64 * 1024

cwltool/docker_uid.py renamed to cwltool/docker_id.py

@@ -2,25 +2,25 @@
 from __future__ import absolute_import
 
 import subprocess
-from typing import List, Text
+from typing import List, Text, Tuple
 
 
-def docker_vm_uid():  # type: () -> int
+def docker_vm_id():  # type: () -> Tuple[int, int]
     """
-    Returns the UID of the default docker user inside the VM
+    Returns the User ID and Group ID of the default docker user inside the VM
 
     When a host is using boot2docker or docker-machine to run docker with
     boot2docker.iso (As on Mac OS X), the UID that mounts the shared filesystem
     inside the VirtualBox VM is likely different than the user's UID on the host.
-    :return: The numeric UID (as a string) of the docker account inside
+    :return: A tuple containing numeric User ID and Group ID of the docker account inside
     the boot2docker VM
     """
     if boot2docker_running():
-        return boot2docker_uid()
+        return boot2docker_id()
     elif docker_machine_running():
-        return docker_machine_uid()
+        return docker_machine_id()
     else:
-        return None
+        return (None, None)
 
 
 def check_output_and_strip(cmd):  # type: (List[Text]) -> Text
@@ -95,23 +95,26 @@ def cmd_output_to_int(cmd):  # type: (List[Text]) -> int
     return None
 
 
-def boot2docker_uid():  # type: () -> int
+def boot2docker_id():  # type: () -> Tuple[int, int]
     """
-    Gets the UID of the docker user inside a running boot2docker vm
-    :return: the UID, or None if error (e.g. boot2docker not present or stopped)
+    Gets the UID and GID of the docker user inside a running boot2docker vm
+    :return: Tuple (UID, GID), or (None, None) if error (e.g. boot2docker not present or stopped)
     """
-    return cmd_output_to_int(['boot2docker', 'ssh', 'id', '-u'])
-
+    uid = cmd_output_to_int(['boot2docker', 'ssh', 'id', '-u'])
+    gid = cmd_output_to_int(['boot2docker', 'ssh', 'id', '-g'])
+    return (uid, gid)
 
-def docker_machine_uid():  # type: () -> int
+def docker_machine_id():  # type: () -> Tuple[int, int]
     """
     Asks docker-machine for active machine and gets the UID of the docker user
     inside the vm
-    :return: the UID, or None if error (e.g. docker-machine not present or stopped)
+    :return: tuple (UID, GID), or (None, None) if error (e.g. docker-machine not present or stopped)
     """
     machine_name = docker_machine_name()
-    return cmd_output_to_int(['docker-machine', 'ssh', machine_name, "id -u"])
+    uid = cmd_output_to_int(['docker-machine', 'ssh', machine_name, "id -u"])
+    gid = cmd_output_to_int(['docker-machine', 'ssh', machine_name, "id -g"])
+    return (uid, gid)
 
 
 if __name__ == '__main__':
-    print(docker_vm_uid())
+    print(docker_vm_id())

cwltool/draft2tool.py

@@ -109,8 +109,9 @@ def revmap_file(builder, outdir, f):
         if f["location"].startswith("file://"):
             path = convert_pathsep_to_unix(uri_file_path(f["location"]))
             revmap_f = builder.pathmapper.reversemap(path)
-            if revmap_f:
-                f["location"] = revmap_f[1]
+            if revmap_f and not builder.pathmapper.mapper(revmap_f[0]).type.startswith("Writable"):
+                f["basename"] = os.path.basename(path)
+                f["location"] = revmap_f[0]
             elif path == builder.outdir:
                 f["location"] = outdir
             elif path.startswith(builder.outdir):
@@ -181,13 +182,14 @@ def __init__(self, toolpath_object, **kwargs):
     def makeJobRunner(self, use_container=True):  # type: (Optional[bool]) -> JobBase
         dockerReq, _ = self.get_requirement("DockerRequirement")
         if not dockerReq and use_container:
-            default_container = self.find_default_container(self)
-            if default_container:
-                self.requirements.insert(0, {
-                    "class": "DockerRequirement",
-                    "dockerPull": default_container
-                })
-                dockerReq = self.requirements[0]
+            if self.find_default_container:
+                default_container = self.find_default_container(self)
+                if default_container:
+                    self.requirements.insert(0, {
+                        "class": "DockerRequirement",
+                        "dockerPull": default_container
+                    })
+                    dockerReq = self.requirements[0]
 
         if dockerReq and use_container:
             return DockerCommandLineJob()
@@ -203,6 +205,17 @@ def makePathMapper(self, reffiles, stagedir, **kwargs):
         # type: (List[Any], Text, **Any) -> PathMapper
         return PathMapper(reffiles, kwargs["basedir"], stagedir)
 
+    def updatePathmap(self, outdir, pathmap, fn):
+        # type: (Text, PathMapper, Dict) -> None
+        if "location" in fn:
+            pathmap.update(fn["location"], pathmap.mapper(fn["location"]).resolved,
+                           os.path.join(outdir, fn["basename"]),
+                           ("Writable" if fn.get("writable") else "") + fn["class"], False)
+        for sf in fn.get("secondaryFiles", []):
+            self.updatePathmap(outdir, pathmap, sf)
+        for ls in fn.get("listing", []):
+            self.updatePathmap(os.path.join(outdir, fn["basename"]), pathmap, ls)
+
     def job(self,
             job_order,  # type: Dict[Text, Text]
             output_callbacks,  # type: Callable[[Any, Any], Any]
@@ -228,9 +241,9 @@ def job(self,
 
             cmdline = flatten(list(map(cachebuilder.generate_arg, cachebuilder.bindings)))
             (docker_req, docker_is_req) = self.get_requirement("DockerRequirement")
-            if docker_req and kwargs.get("use_container") is not False:
+            if docker_req and kwargs.get("use_container"):
                 dockerimg = docker_req.get("dockerImageId") or docker_req.get("dockerPull")
-            elif kwargs.get("default_container", None) is not None and kwargs.get("use_container") is not False:
+            elif kwargs.get("default_container", None) is not None and kwargs.get("use_container"):
                 dockerimg = kwargs.get("default_container")
 
             if dockerimg:
@@ -268,7 +281,7 @@ def job(self,
             jobcachepending = jobcache + ".pending"
 
             if os.path.isdir(jobcache) and not os.path.isfile(jobcachepending):
-                if docker_req and kwargs.get("use_container") is not False:
+                if docker_req and kwargs.get("use_container"):
                     cachebuilder.outdir = kwargs.get("docker_outdir") or "/var/spool/cwl"
                 else:
                     cachebuilder.outdir = jobcache
@@ -327,46 +340,10 @@ def rm_pending_output_callback(output_callbacks, jobcachepending,
         builder.pathmapper = self.makePathMapper(reffiles, builder.stagedir, **make_path_mapper_kwargs)
         builder.requirements = j.requirements
 
-        if _logger.isEnabledFor(logging.DEBUG):
-            _logger.debug(u"[job %s] path mappings is %s", j.name,
-                          json.dumps({p: builder.pathmapper.mapper(p) for p in builder.pathmapper.files()}, indent=4))
-
         _check_adjust = partial(check_adjust, builder)
 
         visit_class([builder.files, builder.bindings], ("File", "Directory"), _check_adjust)
 
-        if self.tool.get("stdin"):
-            with SourceLine(self.tool, "stdin", validate.ValidationException):
-                j.stdin = builder.do_eval(self.tool["stdin"])
-                reffiles.append({"class": "File", "path": j.stdin})
-
-        if self.tool.get("stderr"):
-            with SourceLine(self.tool, "stderr", validate.ValidationException):
-                j.stderr = builder.do_eval(self.tool["stderr"])
-                if os.path.isabs(j.stderr) or ".." in j.stderr:
-                    raise validate.ValidationException("stderr must be a relative path, got '%s'" % j.stderr)
-
-        if self.tool.get("stdout"):
-            with SourceLine(self.tool, "stdout", validate.ValidationException):
-                j.stdout = builder.do_eval(self.tool["stdout"])
-                if os.path.isabs(j.stdout) or ".." in j.stdout or not j.stdout:
-                    raise validate.ValidationException("stdout must be a relative path, got '%s'" % j.stdout)
-
-        if _logger.isEnabledFor(logging.DEBUG):
-            _logger.debug(u"[job %s] command line bindings is %s", j.name, json.dumps(builder.bindings, indent=4))
-
-        dockerReq = self.get_requirement("DockerRequirement")[0]
-        if dockerReq and kwargs.get("use_container"):
-            out_prefix = kwargs.get("tmp_outdir_prefix")
-            j.outdir = kwargs.get("outdir") or tempfile.mkdtemp(prefix=out_prefix)
-            tmpdir_prefix = kwargs.get('tmpdir_prefix')
-            j.tmpdir = kwargs.get("tmpdir") or tempfile.mkdtemp(prefix=tmpdir_prefix)
-            j.stagedir = tempfile.mkdtemp(prefix=tmpdir_prefix)
-        else:
-            j.outdir = builder.outdir
-            j.tmpdir = builder.tmpdir
-            j.stagedir = builder.stagedir
-
         initialWorkdir = self.get_requirement("InitialWorkDirRequirement")[0]
         j.generatefiles = {"class": "Directory", "listing": [], "basename": ""}
         if initialWorkdir:
@@ -402,6 +379,45 @@ def rm_pending_output_callback(output_callbacks, jobcachepending,
                             t["entry"]["writable"] = t.get("writable")
                         ls[i] = t["entry"]
             j.generatefiles[u"listing"] = ls
+            for l in ls:
+                self.updatePathmap(builder.outdir, builder.pathmapper, l)
+            visit_class([builder.files, builder.bindings], ("File", "Directory"), _check_adjust)
+
+        if _logger.isEnabledFor(logging.DEBUG):
+            _logger.debug(u"[job %s] path mappings is %s", j.name,
+                          json.dumps({p: builder.pathmapper.mapper(p) for p in builder.pathmapper.files()}, indent=4))
+
+        if self.tool.get("stdin"):
+            with SourceLine(self.tool, "stdin", validate.ValidationException):
+                j.stdin = builder.do_eval(self.tool["stdin"])
+                reffiles.append({"class": "File", "path": j.stdin})
+
+        if self.tool.get("stderr"):
+            with SourceLine(self.tool, "stderr", validate.ValidationException):
+                j.stderr = builder.do_eval(self.tool["stderr"])
+                if os.path.isabs(j.stderr) or ".." in j.stderr:
+                    raise validate.ValidationException("stderr must be a relative path, got '%s'" % j.stderr)
+
+        if self.tool.get("stdout"):
+            with SourceLine(self.tool, "stdout", validate.ValidationException):
+                j.stdout = builder.do_eval(self.tool["stdout"])
+                if os.path.isabs(j.stdout) or ".." in j.stdout or not j.stdout:
+                    raise validate.ValidationException("stdout must be a relative path, got '%s'" % j.stdout)
+
+        if _logger.isEnabledFor(logging.DEBUG):
+            _logger.debug(u"[job %s] command line bindings is %s", j.name, json.dumps(builder.bindings, indent=4))
+
+        dockerReq = self.get_requirement("DockerRequirement")[0]
+        if dockerReq and kwargs.get("use_container"):
+            out_prefix = kwargs.get("tmp_outdir_prefix")
+            j.outdir = kwargs.get("outdir") or tempfile.mkdtemp(prefix=out_prefix)
+            tmpdir_prefix = kwargs.get('tmpdir_prefix')
+            j.tmpdir = kwargs.get("tmpdir") or tempfile.mkdtemp(prefix=tmpdir_prefix)
+            j.stagedir = tempfile.mkdtemp(prefix=tmpdir_prefix)
+        else:
+            j.outdir = builder.outdir
+            j.tmpdir = builder.tmpdir
+            j.stagedir = builder.stagedir
 
         inplaceUpdateReq = self.get_requirement("http://commonwl.org/cwltool#InplaceUpdateRequirement")[0]
 

cwltool/job.py

@@ -19,7 +19,7 @@
 from .utils import copytree_with_merge, docker_windows_path_adjust, onWindows
 from . import docker
 from .builder import Builder
-from .docker_uid import docker_vm_uid
+from .docker_id import docker_vm_id
 from .errors import WorkflowException
 from .pathmapper import PathMapper
 from .process import (UnsupportedRequirement, empty_subtree, get_feature,
@@ -89,6 +89,7 @@ def deref_links(outputs):  # type: (Any) -> None
         if outputs.get("class") == "File":
             st = os.lstat(outputs["path"])
             if stat.S_ISLNK(st.st_mode):
+                outputs["basename"] = os.path.basename(outputs["path"])
                 outputs["path"] = os.readlink(outputs["path"])
         else:
             for v in outputs.values():
@@ -147,7 +148,7 @@ def _setup(self):  # type: () -> None
 
         for knownfile in self.pathmapper.files():
             p = self.pathmapper.mapper(knownfile)
-            if p.type == "File" and not os.path.isfile(p[0]):
+            if p.type == "File" and not os.path.isfile(p[0]) and p.staged:
                 raise WorkflowException(
                     u"Input file %s (at %s) not found or is not a regular "
                     "file." % (knownfile, self.pathmapper.mapper(knownfile)[0]))
@@ -347,14 +348,14 @@ def run(self, pull_image=True, rm_container=True,
         env = None  # type: MutableMapping[Text, Text]
         try:
             env = cast(MutableMapping[Text, Text], os.environ)
-            if docker_req and kwargs.get("use_container") is not False:
+            if docker_req and kwargs.get("use_container"):
                 img_id = docker.get_from_requirements(docker_req, True, pull_image)
             if img_id is None:
-                find_default_container = self.builder.find_default_container
-                default_container = find_default_container and find_default_container()
-                if default_container:
-                    img_id = default_container
-                    env = cast(MutableMapping[Text, Text], os.environ)
+                if self.builder.find_default_container:
+                    default_container = self.builder.find_default_container()
+                    if default_container:
+                        img_id = default_container
+                        env = cast(MutableMapping[Text, Text], os.environ)
 
             if docker_req and img_id is None and kwargs.get("use_container"):
                 raise Exception("Docker image not available")
@@ -390,13 +391,12 @@ def run(self, pull_image=True, rm_container=True,
         if self.stdout:
             runtime.append("--log-driver=none")
 
-        if onWindows():  # windows os dont have getuid or geteuid functions
-            euid = docker_vm_uid()
-        else:
-            euid = docker_vm_uid() or os.geteuid()
+        euid, egid = docker_vm_id()
+        if not onWindows():  # MS Windows does not have getuid() or geteuid() functions
+            euid, egid = euid or os.geteuid(), egid or os.getgid()
 
-        if kwargs.get("no_match_user", None) is False and euid is not None:
-            runtime.append(u"--user=%s" % (euid))
+        if kwargs.get("no_match_user", None) is False and (euid, egid) != (None, None):
+            runtime.append(u"--user=%d:%d" % (euid, egid))
 
         if rm_container:
             runtime.append(u"--rm")

cwltool/load_tool.py

@@ -22,6 +22,7 @@
 from . import process, update
 from .errors import WorkflowException
 from .process import Process, shortname
+from .update import ALLUPDATES
 
 _logger = logging.getLogger("cwltool")
 
@@ -161,12 +162,17 @@ def validate_document(document_loader,  # type: Loader
     if "cwlVersion" in workflowobj:
         if not isinstance(workflowobj["cwlVersion"], (str, Text)):
             raise Exception("'cwlVersion' must be a string, got %s" % type(workflowobj["cwlVersion"]))
+        if workflowobj["cwlVersion"] not in list(ALLUPDATES):
+            # print out all the Supported Versions of cwlVersion
+            versions = list(ALLUPDATES) # ALLUPDATES is a dict
+            versions.sort()
+            raise ValidationException("'cwlVersion' not valid. Supported CWL versions are: \n{}".format("\n".join(versions)))
         workflowobj["cwlVersion"] = re.sub(
             r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "",
             workflowobj["cwlVersion"])
     else:
-        _logger.warning("No cwlVersion found, treating this file as draft-2.")
-        workflowobj["cwlVersion"] = "draft-2"
+        raise ValidationException("No cwlVersion found."
+            "Use the following syntax in your CWL workflow to declare version: cwlVersion: <version>")
 
     if workflowobj["cwlVersion"] == "draft-2":
         workflowobj = cast(CommentedMap, cmap(update._draft2toDraft3dev1(

cwltool/main.py

@@ -591,7 +591,19 @@ def pathToLoc(p):
             p["location"] = p["path"]
             del p["path"]
 
+    def addSizes(p):
+        if 'location' in p:
+            try:
+                p["size"] = os.stat(p["location"][7:]).st_size  # strip off file://
+            except OSError:
+                pass
+        elif 'contents' in p:
+                p["size"] = len(p['contents'])
+        else:
+            return  # best effort
+
     visit_class(job_order_object, ("File", "Directory"), pathToLoc)
+    visit_class(job_order_object, ("File"), addSizes)
     adjustDirObjs(job_order_object, trim_listing)
     normalizeFilesDirs(job_order_object)
 
@@ -699,7 +711,8 @@ def main(argsl=None,  # type: List[str]
 
         # If On windows platform, A default Docker Container is Used if not explicitely provided by user
         if onWindows() and not args.default_container:
-            args.default_container = "ubuntu"
+            # This docker image is a minimal alpine image with bash installed(size 6 mb). source: https://github.com/frol/docker-alpine-bash
+            args.default_container = "frolvlad/alpine-bash"
 
         # If caller provided custom arguments, it may be not every expected
         # option is set, so fill in no-op defaults to avoid crashing when

cwltool/pathmapper.py

@@ -37,7 +37,7 @@ def visit_class(rec, cls, op):  # type: (Any, Iterable, Union[Callable[..., Any]
     """Apply a function to with "class" in cls."""
 
     if isinstance(rec, dict):
-        if rec.get("class") in cls:
+        if "class" in rec and rec.get("class") in cls:
             op(rec)
         for d in rec:
             visit_class(rec[d], cls, op)
@@ -249,3 +249,6 @@ def reversemap(self, target):  # type: (Text) -> Tuple[Text, Text]
             if v[1] == target:
                 return (k, v[0])
         return None
+
+    def update(self, key, resolved, target, type, stage):  # type: (Text, Text, Text, Text, bool) -> None
+        self._pathmap[key] = MapperEnt(resolved, target, type, stage)