fixed timeout, added num_cpus as argument, handled stdout and stderr

Author: wenting-zhao

commit0/__main__.py

@@ -78,6 +78,7 @@ def main() -> None:
             test_ids,
             config.backend,
             config.timeout,
+            config.num_cpus,
             stdout=True,
         )
     elif command == "evaluate" or command == "evaluate-reference":
@@ -91,6 +92,7 @@ def main() -> None:
             config.branch,
             config.backend,
             config.timeout,
+            config.num_cpus,
             config.num_workers,
         )
     elif command == "save":

commit0/configs/base.yaml

@@ -16,6 +16,7 @@ num_workers: 8
 backend: local
 branch: ai
 timeout: 1_800
+num_cpus: 1
 
 # save related
 github_token: null

commit0/configs/config_class.py

@@ -22,6 +22,7 @@ class Commit0Config:
     branch: str
     # timeout for running pytest
     timeout: int
+    num_cpus: int
 
     # save related
     github_token: Optional[str]

commit0/configs/user.yaml

@@ -2,4 +2,4 @@ defaults:
   - base
   - _self_
 
-backend: local
+backend: modal

commit0/harness/docker_utils.py

@@ -247,7 +247,7 @@ def create_container(
     image_name: str,
     container_name: Optional[str] = None,
     user: Optional[str] = None,
-    command: Optional[str] = None,
+    command: Optional[str] = "tail -f /dev/null",
     nano_cpus: Optional[int] = None,
     logger: Optional[Union[str, logging.Logger]] = None,
 ) -> Container:
@@ -312,7 +312,7 @@ def log_info(x: str) -> None:
             image=image_name,
             name=container_name,
             user=user,
-            command="tail -f /dev/null",
+            command=command,
             nano_cpus=nano_cpus,
             detach=True,
         )

commit0/harness/evaluate.py

@@ -26,6 +26,7 @@ def main(
     branch: str,
     backend: str,
     timeout: int,
+    num_cpus: int,
     num_workers: int,
 ) -> None:
     dataset: Iterator[RepoInstance] = load_dataset(dataset_name, split=dataset_split)  # type: ignore
@@ -52,6 +53,7 @@ def main(
                     test_dir,
                     backend,
                     timeout,
+                    num_cpus,
                     stdout=False,
                 ): None
                 for repo, test_dir in pairs

commit0/harness/execution_context.py

@@ -52,6 +52,7 @@ def __init__(
         spec: Spec,
         logger: logging.Logger,
         timeout: int,
+        num_cpus: int,
         log_dir: Path,
         files_to_copy: Optional[Files] = None,
     ):
@@ -63,6 +64,7 @@ def __init__(
         self.spec = spec
         self.logger = logger
         self.timeout = timeout
+        self.num_cpus = num_cpus
         self.log_dir = log_dir
 
     @abstractmethod
@@ -102,16 +104,18 @@ def __init__(
         spec: Spec,
         logger: logging.Logger,
         timeout: int,
+        num_cpus: int,
         log_dir: Path,
         files_to_copy: Optional[Files] = None,
     ):
-        super().__init__(spec, logger, timeout, log_dir)
+        super().__init__(spec, logger, timeout, num_cpus, log_dir)
 
         self.client = docker.from_env()
         self.container = create_container(
             client=self.client,
             image_name=spec.repo_image_key,
             container_name=spec.get_container_name(),
+            nano_cpus=num_cpus,
             logger=logger,
         )
         self.container.start()
@@ -153,10 +157,11 @@ def __init__(
         spec: Spec,
         logger: logging.Logger,
         timeout: int,
+        num_cpus: int,
         log_dir: Path,
         files_to_copy: Optional[Files] = None,
     ):
-        super().__init__(spec, logger, timeout, log_dir)
+        super().__init__(spec, logger, timeout, num_cpus, log_dir)
 
         self.app = modal.App()
 
@@ -181,19 +186,22 @@ def exec_run_with_timeout(self, command: str) -> tuple[str, bool, float]:
                 "-c",
                 f"{command} && cp {str(report_file)} /vol/report.json",
                 image=self.image,
-                cpu=4.0,
+                cpu=self.num_cpus,
                 timeout=self.timeout,
                 app=self.app,
                 volumes={"/vol": vol},
             )
             self.sandbox.wait()
 
-            stdout = read_stream(self.sandbox.stdout)
-            # stderr = read_stream(self.sandbox.stderr)
+            # stdout has been redirected to stderr
+            stdout = read_stream(self.sandbox.stderr)
 
-            # return_code = self.sandbox.returncode
-            # maybe use return_code for timeout info?
-            # i dont know if sandboxes throw modal.TimeoutError
+            return_code = self.sandbox.returncode
+            # https://github.com/modal-labs/modal-client/blob/d577b2916b5c3bf4ebbcb58fadced84d85e1cf8c/modal/sandbox.py#L413
+            if return_code == 124:
+                timed_out = True
+            else:
+                timed_out = False
 
             # copy over report.json from mount
             with (self.log_dir / "report.json").open("wb") as f:
@@ -204,8 +212,7 @@ def exec_run_with_timeout(self, command: str) -> tuple[str, bool, float]:
 
             end_time = time.time()
 
-            # TODO: add check for timeout
-            return stdout, False, end_time - start_time
+            return stdout, timed_out, end_time - start_time
 
     def __exit__(
         self,

commit0/harness/run_pytest_ids.py

@@ -36,6 +36,7 @@ def main(
     test_ids: str,
     backend: str,
     timeout: int,
+    num_cpus: int,
     stdout: bool,
 ) -> str:
     """Runs the pytests for repos in a dataset.
@@ -93,7 +94,7 @@ def main(
 
     try:
         with execution_context(
-            spec, logger, timeout, log_dir, files_to_copy
+            spec, logger, timeout, num_cpus, log_dir, files_to_copy
         ) as context:
             output, timed_out, total_runtime = context.exec_run_with_timeout(
                 "/bin/bash /eval.sh"
@@ -103,7 +104,8 @@ def main(
                 output, "--json-report --json-report-file=report.json"
             )
             context.write_test_output(test_output, timed_out)
-            print(test_output)
+            if stdout:
+                print(test_output)
     except EvaluationError as e:
         error_msg = (
             f"Error in running pytest for {repo}: {e}\n"

commit0/harness/spec.py

@@ -157,6 +157,8 @@ def make_eval_script_list(instance: RepoInstance, repo_directory: str) -> list[s
         f"git reset --hard {instance['base_commit']}",
         "git status",
     ]
+    for i in range(len(eval_script_list)):
+        eval_script_list[i] = f"{eval_script_list[i]} 1>&2"
     return eval_script_list