Skip to content

Commit f081a89

Browse files
mr-cmr-c
committed
remove dry-run, add tmp-outdir-prefix to docker
1 parent 2d97546 commit f081a89

File tree

3 files changed

+57
-49
lines changed

3 files changed

+57
-49
lines changed

‎cwltool/docker.py

Lines changed: 38 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,11 @@ def _check_docker_machine_path(path): # type: (Text) -> None
6767
class DockerCommandLineJob(ContainerCommandLineJob):
6868

6969
@staticmethod
70-
def get_image(dockerRequirement, pull_image, dry_run=False, force_pull=False):
71-
# type: (Dict[Text, Text], bool, bool, bool) -> bool
70+
def get_image(dockerRequirement, # type: Dict[Text, Text]
71+
pull_image, # type: bool
72+
force_pull=False, # type: bool
73+
tmp_outdir_prefix=None # type: Text
74+
): # type: (...) -> bool
7275
found = False
7376

7477
if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
@@ -108,57 +111,58 @@ def get_image(dockerRequirement, pull_image, dry_run=False, force_pull=False):
108111
if "dockerPull" in dockerRequirement:
109112
cmd = ["docker", "pull", str(dockerRequirement["dockerPull"])]
110113
_logger.info(Text(cmd))
111-
if not dry_run:
112-
subprocess.check_call(cmd, stdout=sys.stderr)
113-
found = True
114+
subprocess.check_call(cmd, stdout=sys.stderr)
115+
found = True
114116
elif "dockerFile" in dockerRequirement:
115-
dockerfile_dir = str(tempfile.mkdtemp())
117+
dockerfile_dir = str(tempfile.mkdtemp(prefix=tmp_outdir_prefix))
116118
with open(os.path.join(dockerfile_dir, "Dockerfile"), "wb") as df:
117119
df.write(dockerRequirement["dockerFile"].encode('utf-8'))
118120
cmd = ["docker", "build", "--tag=%s" %
119121
str(dockerRequirement["dockerImageId"]), dockerfile_dir]
120122
_logger.info(Text(cmd))
121-
if not dry_run:
122-
subprocess.check_call(cmd, stdout=sys.stderr)
123-
found = True
123+
subprocess.check_call(cmd, stdout=sys.stderr)
124+
found = True
124125
elif "dockerLoad" in dockerRequirement:
125126
cmd = ["docker", "load"]
126127
_logger.info(Text(cmd))
127-
if not dry_run:
128-
if os.path.exists(dockerRequirement["dockerLoad"]):
129-
_logger.info(u"Loading docker image from %s", dockerRequirement["dockerLoad"])
130-
with open(dockerRequirement["dockerLoad"], "rb") as f:
131-
loadproc = subprocess.Popen(cmd, stdin=f, stdout=sys.stderr)
132-
else:
133-
loadproc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=sys.stderr)
134-
_logger.info(u"Sending GET request to %s", dockerRequirement["dockerLoad"])
135-
req = requests.get(dockerRequirement["dockerLoad"], stream=True)
136-
n = 0
137-
for chunk in req.iter_content(1024 * 1024):
138-
n += len(chunk)
139-
_logger.info("\r%i bytes" % (n))
140-
loadproc.stdin.write(chunk)
141-
loadproc.stdin.close()
142-
rcode = loadproc.wait()
143-
if rcode != 0:
144-
raise WorkflowException("Docker load returned non-zero exit status %i" % (rcode))
145-
found = True
128+
if os.path.exists(dockerRequirement["dockerLoad"]):
129+
_logger.info(u"Loading docker image from %s", dockerRequirement["dockerLoad"])
130+
with open(dockerRequirement["dockerLoad"], "rb") as f:
131+
loadproc = subprocess.Popen(cmd, stdin=f, stdout=sys.stderr)
132+
else:
133+
loadproc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=sys.stderr)
134+
_logger.info(u"Sending GET request to %s", dockerRequirement["dockerLoad"])
135+
req = requests.get(dockerRequirement["dockerLoad"], stream=True)
136+
n = 0
137+
for chunk in req.iter_content(1024 * 1024):
138+
n += len(chunk)
139+
_logger.info("\r%i bytes" % (n))
140+
loadproc.stdin.write(chunk)
141+
loadproc.stdin.close()
142+
rcode = loadproc.wait()
143+
if rcode != 0:
144+
raise WorkflowException("Docker load returned non-zero exit status %i" % (rcode))
145+
found = True
146146
elif "dockerImport" in dockerRequirement:
147147
cmd = ["docker", "import", str(dockerRequirement["dockerImport"]),
148148
str(dockerRequirement["dockerImageId"])]
149149
_logger.info(Text(cmd))
150-
if not dry_run:
151-
subprocess.check_call(cmd, stdout=sys.stderr)
152-
found = True
150+
subprocess.check_call(cmd, stdout=sys.stderr)
151+
found = True
153152

154153
if found:
155154
with found_images_lock:
156155
found_images.add(dockerRequirement["dockerImageId"])
157156

158157
return found
159158

160-
def get_from_requirements(self, r, req, pull_image, dry_run=False, force_pull=False):
161-
# type: (Dict[Text, Text], bool, bool, bool, bool) -> Text
159+
def get_from_requirements(self,
160+
r, # type: Dict[Text, Text]
161+
req, # type: bool
162+
pull_image, # type: bool
163+
force_pull=False, # type: bool
164+
tmp_outdir_prefix=None # type: Text
165+
): # type: (...) -> Text
162166
if r:
163167
errmsg = None
164168
try:
@@ -174,7 +178,7 @@ def get_from_requirements(self, r, req, pull_image, dry_run=False, force_pull=Fa
174178
else:
175179
return None
176180

177-
if self.get_image(r, pull_image, dry_run, force_pull=force_pull):
181+
if self.get_image(r, pull_image, force_pull, tmp_outdir_prefix):
178182
return r["dockerImageId"]
179183
else:
180184
if req:

‎cwltool/job.py

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -341,8 +341,13 @@ class ContainerCommandLineJob(JobBase):
341341
__metaclass__ = ABCMeta
342342

343343
@abstractmethod
344-
def get_from_requirements(self, r, req, pull_image, dry_run=False, force_pull=False):
345-
# type: (Dict[Text, Text], bool, bool, bool, bool) -> Text
344+
def get_from_requirements(self,
345+
r, # type: Dict[Text, Text]
346+
req, # type: bool
347+
pull_image, # type: bool
348+
force_pull=False, # type: bool
349+
tmp_outdir_prefix=None # type: Text
350+
): # type: (...) -> Text
346351
pass
347352

348353
@abstractmethod
@@ -377,7 +382,9 @@ def run(self, pull_image=True, rm_container=True,
377382
try:
378383
env = cast(MutableMapping[Text, Text], os.environ)
379384
if docker_req and kwargs.get("use_container"):
380-
img_id = str(self.get_from_requirements(docker_req, True, pull_image, force_pull=kwargs.get("force_docker_pull")))
385+
img_id = str(self.get_from_requirements(docker_req, True,
386+
pull_image, kwargs.get("force_docker_pull"),
387+
kwargs.get('tmp_outdir_prefix')))
381388
if img_id is None:
382389
if self.builder.find_default_container:
383390
default_container = self.builder.find_default_container()

‎cwltool/singularity.py

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,6 @@ class SingularityCommandLineJob(ContainerCommandLineJob):
5050
@staticmethod
5151
def get_image(dockerRequirement, # type: Dict[Text, Text]
5252
pull_image, # type: bool
53-
dry_run=False, # type: bool
5453
force_pull=False # type: bool
5554
):
5655
# type: (...) -> bool
@@ -95,9 +94,8 @@ def get_image(dockerRequirement, # type: Dict[Text, Text]
9594
str(dockerRequirement["dockerImageId"]),
9695
str(dockerRequirement["dockerPull"])]
9796
_logger.info(Text(cmd))
98-
if not dry_run:
99-
check_call(cmd, stdout=sys.stderr)
100-
found = True
97+
check_call(cmd, stdout=sys.stderr)
98+
found = True
10199
elif "dockerFile" in dockerRequirement:
102100
raise WorkflowException(SourceLine(
103101
dockerRequirement, 'dockerFile').makeError(
@@ -117,13 +115,12 @@ def get_image(dockerRequirement, # type: Dict[Text, Text]
117115
return found
118116

119117
def get_from_requirements(self,
120-
r, # type: Optional[Dict[Text, Text]]
121-
req, # type: bool
122-
pull_image, # type: bool
123-
dry_run=False, # type: bool
124-
force_pull=False # type: bool
125-
):
126-
# type: (...) -> Text
118+
r, # type: Optional[Dict[Text, Text]]
119+
req, # type: bool
120+
pull_image, # type: bool
121+
force_pull=False, # type: bool
122+
tmp_outdir_prefix=None # type: Text
123+
): # type: (...) -> Text
127124
"""
128125
Returns the filename of the Singularity image (e.g.
129126
hello-world-latest.img).
@@ -144,7 +141,7 @@ def get_from_requirements(self,
144141
else:
145142
return None
146143

147-
if self.get_image(r, pull_image, dry_run, force_pull):
144+
if self.get_image(r, pull_image, force_pull):
148145
return os.path.abspath(r["dockerImageId"])
149146
else:
150147
if req:

0 commit comments

Comments
 (0)