Skip to content

Commit 5e4ed74

Browse files
authored
Merge pull request #92 from common-workflow-language/stagefiles
Pathmapper now has list of file objects and not just file names.
2 parents c8f5848 + 3902ed8 commit 5e4ed74

File tree

110 files changed

+1156
-584
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

110 files changed

+1156
-584
lines changed

cwltool/builder.py

Lines changed: 8 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
from typing import Any, Union, AnyStr, Callable
77
from .errors import WorkflowException
88
from .stdfsaccess import StdFsAccess
9-
from .pathmapper import PathMapper
9+
from .pathmapper import PathMapper, adjustFileObjs, adjustDirObjs
1010

1111
CONTENT_LIMIT = 64 * 1024
1212

@@ -17,18 +17,6 @@ def substitute(value, replace): # type: (str, str) -> str
1717
else:
1818
return value + replace
1919

20-
def adjustFileObjs(rec, op): # type: (Any, Callable[[Any], Any]) -> None
21-
"""Apply an update function to each File object in the object `rec`."""
22-
23-
if isinstance(rec, dict):
24-
if rec.get("class") == "File":
25-
op(rec)
26-
for d in rec:
27-
adjustFileObjs(rec[d], op)
28-
if isinstance(rec, list):
29-
for d in rec:
30-
adjustFileObjs(d, op)
31-
3220
class Builder(object):
3321

3422
def __init__(self): # type: () -> None
@@ -114,7 +102,7 @@ def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]):
114102
if schema["type"] == "File":
115103
self.files.append(datum)
116104
if binding and binding.get("loadContents"):
117-
with self.fs_access.open(datum["path"], "rb") as f:
105+
with self.fs_access.open(datum["location"], "rb") as f:
118106
datum["contents"] = f.read(CONTENT_LIMIT)
119107

120108
if "secondaryFiles" in schema:
@@ -124,11 +112,11 @@ def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]):
124112
if isinstance(sf, dict) or "$(" in sf or "${" in sf:
125113
secondary_eval = self.do_eval(sf, context=datum)
126114
if isinstance(secondary_eval, basestring):
127-
sfpath = {"path": secondary_eval, "class": "File"}
115+
sfpath = {"location": secondary_eval, "class": "File"}
128116
else:
129117
sfpath = secondary_eval
130118
else:
131-
sfpath = {"path": substitute(datum["path"], sf), "class": "File"}
119+
sfpath = {"location": substitute(datum["location"], sf), "class": "File"}
132120
if isinstance(sfpath, list):
133121
datum["secondaryFiles"].extend(sfpath)
134122
else:
@@ -140,6 +128,10 @@ def _capture_files(f):
140128

141129
adjustFileObjs(datum.get("secondaryFiles", []), _capture_files)
142130

131+
if schema["type"] == "Directory":
132+
self.files.append(datum)
133+
134+
143135
# Position to front of the sort key
144136
if binding:
145137
for bi in bindings:

cwltool/cwltest.py

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,30 @@ def compare(a, b): # type: (Any, Any) -> bool
3737
# ignore empty collections
3838
b = {k: v for k, v in b.iteritems()
3939
if not isinstance(v, (list, dict)) or len(v) > 0}
40+
elif a.get("class") == "Directory":
41+
if len(a["listing"]) != len(b["listing"]):
42+
return False
43+
for i in a["listing"]:
44+
found = False
45+
for j in b["listing"]:
46+
try:
47+
if compare(i, j):
48+
found = True
49+
break
50+
except:
51+
pass
52+
if not found:
53+
raise CompareFail(u"%s not in %s" % (json.dumps(i, indent=4, sort_keys=True), json.dumps(b, indent=4, sort_keys=True)))
54+
55+
a = {k: v for k, v in a.iteritems()
56+
if k not in ("path", "location", "listing")}
57+
b = {k: v for k, v in b.iteritems()
58+
if k not in ("path", "location", "listing")}
59+
4060
if len(a) != len(b):
4161
raise CompareFail(u"expected %s\ngot %s" % (json.dumps(a, indent=4, sort_keys=True), json.dumps(b, indent=4, sort_keys=True)))
4262
for c in a:
43-
if a.get("class") != "File" or c != "path":
63+
if a.get("class") != "File" or c not in ("path", "location", "listing"):
4464
if c not in b:
4565
raise CompareFail(u"%s not in %s" % (c, b))
4666
if not compare(a[c], b[c]):

0 commit comments

Comments
 (0)