Merge branch 'master' into pack

Conflicts:
	cwltool/main.py
	cwltool/process.py

Author: Peter Amstutz

cwltool/draft2tool.py

@@ -52,9 +52,9 @@ def run(self, **kwargs):  # type: (**Any) -> None
                 _logger.warn(u"Failed to evaluate expression:\n%s", e, exc_info=(e if kwargs.get('debug') else False))
                 self.output_callback({}, "permanentFail")
 
-    def job(self, joborder, input_basedir, output_callback, **kwargs):
+    def job(self, joborder, output_callback, **kwargs):
         # type: (Dict[str,str], str, Callable[[Any, Any], Any], **Any) -> Generator[ExpressionTool.ExpressionJob, None, None]
-        builder = self._init_job(joborder, input_basedir, **kwargs)
+        builder = self._init_job(joborder, **kwargs)
 
         j = ExpressionTool.ExpressionJob()
         j.builder = builder
@@ -119,19 +119,19 @@ def __init__(self, toolpath_object, **kwargs):
     def makeJobRunner(self):  # type: () -> CommandLineJob
         return CommandLineJob()
 
-    def makePathMapper(self, reffiles, input_basedir, **kwargs):
+    def makePathMapper(self, reffiles, **kwargs):
         # type: (Set[str], str, **Any) -> PathMapper
         dockerReq, _ = self.get_requirement("DockerRequirement")
         try:
             if dockerReq and kwargs.get("use_container"):
-                return DockerPathMapper(reffiles, input_basedir)
+                return DockerPathMapper(reffiles, kwargs["basedir"])
             else:
-                return PathMapper(reffiles, input_basedir)
+                return PathMapper(reffiles, kwargs["basedir"])
         except OSError as e:
             if e.errno == errno.ENOENT:
                 raise WorkflowException(u"Missing input file %s" % e)
 
-    def job(self, joborder, input_basedir, output_callback, **kwargs):
+    def job(self, joborder, output_callback, **kwargs):
         # type: (Dict[str,str], str, Callable[..., Any], **Any) -> Generator[Union[CommandLineJob, CallbackJob], None, None]
 
         jobname = uniquename(kwargs.get("name", shortname(self.tool.get("id", "job"))))
@@ -140,9 +140,9 @@ def job(self, joborder, input_basedir, output_callback, **kwargs):
             cacheargs = kwargs.copy()
             cacheargs["outdir"] = "/out"
             cacheargs["tmpdir"] = "/tmp"
-            cachebuilder = self._init_job(joborder, input_basedir, **cacheargs)
+            cachebuilder = self._init_job(joborder, **cacheargs)
             cachebuilder.pathmapper = PathMapper(set((f["path"] for f in cachebuilder.files)),
-                                                 input_basedir)
+                                                 kwargs["basedir"])
 
             cmdline = flatten(map(cachebuilder.generate_arg, cachebuilder.bindings))
             (docker_req, docker_is_req) = self.get_requirement("DockerRequirement")
@@ -198,7 +198,7 @@ def rm_pending_output_callback(output_callback, jobcachepending,
                         partial(rm_pending_output_callback, output_callback,
                             jobcachepending))
 
-        builder = self._init_job(joborder, input_basedir, **kwargs)
+        builder = self._init_job(joborder, **kwargs)
 
         reffiles = set((f["path"] for f in builder.files))
 
@@ -232,7 +232,7 @@ def rm_pending_output_callback(output_callback, jobcachepending,
             if os.path.isabs(j.stdout) or ".." in j.stdout:
                 raise validate.ValidationException("stdout must be a relative path")
 
-        builder.pathmapper = self.makePathMapper(reffiles, input_basedir, **kwargs)
+        builder.pathmapper = self.makePathMapper(reffiles, **kwargs)
         builder.requirements = j.requirements
 
         # map files to assigned path inside a container. We need to also explicitly
@@ -298,7 +298,7 @@ def collect_output_ports(self, ports, builder, outdir):
             custom_output = os.path.join(outdir, "cwl.output.json")
             if builder.fs_access.exists(custom_output):
                 with builder.fs_access.open(custom_output, "r") as f:
-                    ret = yaml.load(f)
+                    ret = json.load(f)
                 _logger.debug(u"Raw output from %s: %s", custom_output, json.dumps(ret, indent=4))
                 adjustFileObjs(ret, remove_hostfs)
                 adjustFileObjs(ret,
@@ -338,7 +338,9 @@ def collect_output(self, schema, builder, outdir):
                         globpatterns.extend(aslist(gb))
 
                 for gb in globpatterns:
-                    if gb.startswith("/"):
+                    if gb.startswith(outdir):
+                        gb = gb[len(outdir)+1:]
+                    elif gb.startswith("/"):
                         raise WorkflowException("glob patterns must not start with '/'")
                     try:
                         r.extend([{"path": g, "class": "File", "hostfs": True}

cwltool/factory.py

@@ -13,7 +13,9 @@ def __init__(self, t, factory):  # type: (Process, Factory) -> None
         self.factory = factory
 
     def __call__(self, **kwargs):  # type: (**Any) -> Union[str,Dict[str,str]]
-        return self.factory.executor(self.t, kwargs, os.getcwd(), None, **self.factory.execkwargs)
+        execkwargs = self.factory.execkwargs.copy()
+        execkwargs["basedir"] = os.getcwd()
+        return self.factory.executor(self.t, kwargs, **execkwargs)
 
 class Factory(object):
     def __init__(self, makeTool=workflow.defaultMakeTool,

cwltool/job.py

@@ -70,7 +70,11 @@ def run(self, dry_run=False, pull_image=True, rm_container=True,
         #    json.dump(self.joborder, fp)
 
         runtime = []  # type: List[unicode]
-        env = {"TMPDIR": self.tmpdir}  # type: Mapping[str,str]
+
+        # spec currently says "HOME must be set to the designated output
+        # directory." but spec might change to designated temp directory.
+        # env = {"TMPDIR": self.tmpdir, "HOME": self.tmpdir}  # type: Mapping[str,str]
+        env = {"TMPDIR": self.tmpdir, "HOME": self.outdir}  # type: Mapping[str,str]
 
         (docker_req, docker_is_req) = get_feature(self, "DockerRequirement")
 
@@ -112,6 +116,11 @@ def run(self, dry_run=False, pull_image=True, rm_container=True,
 
             runtime.append("--env=TMPDIR=/tmp")
 
+            # spec currently says "HOME must be set to the designated output
+            # directory." but spec might change to designated temp directory.
+            # runtime.append("--env=HOME=/tmp")
+            runtime.append("--env=HOME=/var/spool/cwl")
+
             for t,v in self.environment.items():
                 runtime.append(u"--env=%s=%s" % (t, v))
 
@@ -140,7 +149,7 @@ def run(self, dry_run=False, pull_image=True, rm_container=True,
         _logger.info(u"[job %s] %s$ %s%s%s",
                      self.name,
                      self.outdir,
-                     " ".join([shellescape.quote(str(arg)) if shouldquote(str(arg)) else str(arg) for arg in (runtime + self.command_line)]),
+                     " \\\n    ".join([shellescape.quote(str(arg)) if shouldquote(str(arg)) else str(arg) for arg in (runtime + self.command_line)]),
                      u' < %s' % (self.stdin) if self.stdin else '',
                      u' > %s' % os.path.join(self.outdir, self.stdout) if self.stdout else '')