Update scandeps to produce nested directory listing to correctly capture file

structure.

Author: Peter Amstutz

cwltool/main.py

@@ -26,7 +26,7 @@
 from . import workflow
 from .errors import WorkflowException, UnsupportedRequirement
 from .cwlrdf import printrdf, printdot
-from .process import shortname, Process, getListing, relocateOutputs, cleanIntermediate
+from .process import shortname, Process, getListing, relocateOutputs, cleanIntermediate, scandeps
 from .load_tool import fetch_document, validate_document, make_tool
 from . import draft2tool
 from .builder import adjustFileObjs, adjustDirObjs
@@ -423,29 +423,29 @@ def pathToLoc(p):
     return (job_order_object, input_basedir)
 
 
-def printdeps(obj, document_loader, stdout, relative_deps, basedir=None):
+def printdeps(obj, document_loader, stdout, relative_deps, uri, basedir=None):
     # type: (Dict[unicode, Any], Loader, IO[Any], bool, str) -> None
     deps = {"class": "File",
-            "path": obj.get("id", "#")}
+            "location": uri}
 
     def loadref(b, u):
         return document_loader.resolve_ref(u, base_url=b)[0]
 
-    sf = scandeps(basedir if basedir else obj["id"], obj,
+    sf = scandeps(basedir if basedir else uri, obj,
                           set(("$import", "run")),
-                          set(("$include", "$schemas", "path")), loadref)
+                          set(("$include", "$schemas", "path", "location")), loadref)
     if sf:
         deps["secondaryFiles"] = sf
 
     if relative_deps:
         if relative_deps == "primary":
-            base = basedir if basedir else os.path.dirname(obj["id"])
+            base = basedir if basedir else os.path.dirname(uri)
         elif relative_deps == "cwd":
             base = "file://" + os.getcwd()
         else:
             raise Exception(u"Unknown relative_deps %s" % relative_deps)
         def makeRelative(ob):
-            u = ob["location"]
+            u = ob.get("location", ob.get("path"))
             if ":" in u.split("/")[0] and not u.startswith("file://"):
                 pass
             else:
@@ -604,7 +604,7 @@ def main(argsl=None,
             document_loader, workflowobj, uri = fetch_document(args.workflow)
 
             if args.print_deps:
-                printdeps(workflowobj, document_loader, stdout, args.relative_deps)
+                printdeps(workflowobj, document_loader, stdout, args.relative_deps, uri)
                 return 0
 
             document_loader, avsc_names, processobj, metadata, uri \

cwltool/process.py

@@ -549,6 +549,41 @@ def uniquename(stem):  # type: (unicode) -> unicode
     _names.add(u)
     return u
 
+def nestdir(base, deps):
+    dirname = os.path.dirname(base) + "/"
+    subid = deps["location"]
+    if subid.startswith(dirname):
+        s2 = subid[len(dirname):]
+        sp = s2.split('/')
+        sp.pop()
+        while sp:
+            nx = sp.pop()
+            deps = {
+                "entryname": nx,
+                "entry": {
+                    "class": "Directory",
+                    "listing": [deps]
+                }
+            }
+    return deps
+
+def mergedirs(listing):
+    r = []
+    ents = {}
+    for e in listing:
+        if "entryname" in e:
+            if e["entryname"] not in ents:
+                ents[e["entryname"]] = e
+            elif e["entry"]["class"] == "Directory":
+                ents[e["entryname"]]["entry"]["listing"].extend(e["entry"]["listing"])
+        else:
+            r.append(e)
+    for e in ents.itervalues():
+        if e["entry"]["class"] == "Directory":
+            e["entry"]["listing"] = mergedirs(e["entry"]["listing"])
+    r.extend(ents.itervalues())
+    return r
+
 def scandeps(base, doc, reffields, urlfields, loadref):
     # type: (unicode, Any, Set[str], Set[str], Callable[[unicode, str], Any]) -> List[Dict[str, str]]
     r = []
@@ -574,20 +609,25 @@ def scandeps(base, doc, reffields, urlfields, loadref):
                         deps = {
                             "class": "File",
                             "location": subid
-                            }  # type: Dict[str, Any]
+                        }  # type: Dict[str, Any]
                         sf = scandeps(subid, sub, reffields, urlfields, loadref)
                         if sf:
                             deps["secondaryFiles"] = sf
+                        deps = nestdir(base, deps)
                         r.append(deps)
             elif k in urlfields:
                 for u in aslist(v):
-                    r.append({
+                    deps = {
                         "class": "File",
                         "location": urlparse.urljoin(base, u)
-                    })
+                    }
+                    deps = nestdir(base, deps)
+                    r.append(deps)
             else:
                 r.extend(scandeps(base, v, reffields, urlfields, loadref))
     elif isinstance(doc, list):
         for d in doc:
             r.extend(scandeps(base, d, reffields, urlfields, loadref))
+
+    r = mergedirs(r)
     return r