Merge pull request #741 from wtsi-hgi/cache_fix

Fix #740 (cachedir not working in certain situations)

Author: mr-c

.travis.yml

@@ -1,4 +1,6 @@
-sudo: false
+sudo: true
+services:
+  - docker
 language: python
 cache: pip
 python:

cwltool/command_line_tool.py

@@ -296,7 +296,7 @@ def job(self,
                            "EnvVarRequirement",
                            "CreateFileRequirement",
                            "ShellCommandRequirement"}
-            for rh in (self.requirements, self.hints):
+            for rh in (self.original_requirements, self.original_hints):
                 for r in reversed(rh):
                     if r["class"] in interesting and r["class"] not in keydict:
                         keydict[r["class"]] = r

cwltool/process.py

@@ -17,6 +17,7 @@
 from functools import cmp_to_key
 from typing import (Any, Callable, Dict, Generator, List, Set, Text,
                     Tuple, Union, cast, Optional)
+import copy
 
 import schema_salad.schema as schema
 import schema_salad.validate as validate
@@ -453,6 +454,9 @@ def __init__(self, toolpath_object, **kwargs):
                              self.tool.get("requirements", []) +
                              get_overrides(kwargs.get("overrides", []), self.tool["id"]).get("requirements", []))
         self.hints = kwargs.get("hints", []) + self.tool.get("hints", [])
+        # Versions of requirements and hints which aren't mutated.
+        self.original_requirements = copy.deepcopy(self.requirements)
+        self.original_hints = copy.deepcopy(self.hints)
         self.formatgraph = None  # type: Graph
         if "loader" in kwargs:
             self.formatgraph = kwargs["loader"].graph

tests/test_examples.py

@@ -6,6 +6,8 @@
 import sys
 import json
 import logging
+import tempfile
+import shutil
 
 from io import StringIO, BytesIO
 
@@ -651,13 +653,30 @@ def test_no_js_console(self):
 
 @needs_docker
 class TestCache(TestCmdLine):
+    def setUp(self):
+        self.cache_dir = tempfile.mkdtemp("cwltool_cache")
+
+    def tearDown(self):
+        shutil.rmtree(self.cache_dir)
+
     def test_wf_without_container(self):
         test_file = "hello-workflow.cwl"
-        error_code, stdout, stderr = self.get_main_output(["--cachedir", "cache",
+        error_code, stdout, stderr = self.get_main_output(["--cachedir", self.cache_dir,
                                                    get_data("tests/wf/" + test_file), "--usermessage", "hello"])
         self.assertIn("completed success", stderr)
         self.assertEquals(error_code, 0)
 
+    def test_issue_740_fixed(self):
+        test_file = "cache_test_workflow.cwl"
+        error_code, stdout, stderr = self.get_main_output(["--cachedir", self.cache_dir, get_data("tests/wf/" + test_file)])
+        self.assertIn("completed success", stderr)
+        self.assertEquals(error_code, 0)
+
+        error_code, stdout, stderr = self.get_main_output(["--cachedir", self.cache_dir, get_data("tests/wf/" + test_file)])
+        self.assertNotIn("Output of job will be cached in", stderr)
+        self.assertEquals(error_code, 0)
+
+
 @needs_docker
 class TestChecksum(TestCmdLine):
 

tests/wf/cache_test_workflow.cwl

@@ -0,0 +1,21 @@
+#!/usr/bin/env cwl-runner
+
+cwlVersion: v1.0
+class: Workflow
+
+inputs: []
+outputs: []
+
+steps:
+  task1:
+    run: touch_tool.cwl
+    in:
+      message:
+         default: one
+    out: []
+  task2:
+    run: touch_tool.cwl
+    in:
+      message:
+         default: two
+    out: []

tests/wf/touch_tool.cwl

@@ -0,0 +1,18 @@
+#!/usr/bin/env cwl-runner
+
+cwlVersion: v1.0
+class: CommandLineTool
+baseCommand: touch
+requirements:
+  DockerRequirement:
+    dockerPull: alpine
+inputs:
+  message:
+    type: string
+    inputBinding:
+      position: 1
+outputs:
+  - id: out
+    type: File
+    outputBinding:
+      glob: "*"