.

Author: srush

commit0/__init__.py

@@ -1 +1,3 @@
+"""Commit0 Lib"""
+
 __version__ = "0.0.1"

commit0/harness/build.py

@@ -3,7 +3,7 @@
 
 import docker
 from datasets import load_dataset
-
+from typing import Iterator
 from commit0.harness.docker_build import build_repo_images
 from commit0.harness.spec import make_spec
 
@@ -18,7 +18,7 @@ def main(
     base_dir: str,
     config_file: str,
 ) -> None:
-    dataset = load_dataset(hf_name, split="test")
+    dataset: Iterator[RepoInstance] = load_dataset(hf_name, split="test")
     specs = []
     for example in dataset:
         spec = make_spec(example)
@@ -30,7 +30,12 @@ def main(
 
 
 def add_init_args(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument("--hf_name", type=str, help="HF dataset name")
+    parser.add_argument(
+        "--hf_name",
+        type=str,
+        help="HF dataset name",
+        default="wentingzhao/commit0_docstring",
+    )
     parser.add_argument(
         "--base_dir",
         type=str,

commit0/harness/constants.py

@@ -8,6 +8,7 @@ class RepoInstance(TypedDict):
     base_commit: str
     reference_commit: str
     setup: dict
+    test: str
 
 
 # Constants - Evaluation Log Directories

commit0/harness/docker_build.py

@@ -2,6 +2,7 @@
 import re
 import traceback
 import docker
+import docker.errors
 from tqdm import tqdm
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
@@ -296,3 +297,6 @@ def build_repo_images(
 
     # Return the list of (un)successfuly built images
     return successful, failed
+
+
+__all__ = []

commit0/harness/docker_utils.py

@@ -11,6 +11,7 @@
 from pathlib import Path
 from io import BytesIO
 from typing import Optional, List, Union
+import docker.errors
 
 from docker.models.containers import Container
 
@@ -107,7 +108,7 @@ def safe_extract(
 
             tar.extractall(path, members, numeric_owner=numeric_owner)
 
-        safe_extract(tar, path=dst.parent)
+        safe_extract(tar, path=str(dst.parent))
 
     # Move the extracted file to desired dst path if tar extraction gives src.name
     extracted_file_path = dst.parent / src.name
@@ -192,7 +193,7 @@ def write_to_container(container: Container, data: str, dst: Path) -> None:
 def cleanup_container(
     client: docker.DockerClient,
     container: docker.Container,
-    logger: Union[str, logging.Logger],
+    logger: Union[None, str, logging.Logger],
 ) -> None:
     """Stop and remove a Docker container.
     Performs this forcefully if the container cannot be stopped with the python API.
@@ -211,8 +212,12 @@ def cleanup_container(
 
     if not logger:
         # if logger is None, print to stdout
-        log_error = print
-        log_info = print
+        def log_error(x: str) -> None:
+            print(x)
+
+        def log_info(x: str) -> None:
+            print(x)
+
         raise_error = True
     elif logger == "quiet":
         # if logger is "quiet", don't print anything
@@ -224,9 +229,15 @@ def log_error(x: str) -> None:
 
         raise_error = True
     else:
+        assert isinstance(logger, logging.Logger)
+
         # if logger is a logger object, use it
-        log_error = logger.info
-        log_info = logger.info
+        def log_error(x: str) -> None:
+            logger.info(x)
+
+        def log_info(x: str) -> None:
+            logger.info(x)
+
         raise_error = False
 
     # Attempt to stop the container
@@ -276,11 +287,11 @@ def log_error(x: str) -> None:
 def create_container(
     client: docker.DockerClient,
     image_name: str,
-    container_name: str = None,
-    user: str = None,
-    command: str = None,
-    nano_cpus: int = None,
-    logger: Union[str, logging.Logger] = None,
+    container_name: Optional[str] = None,
+    user: Optional[str] = None,
+    command: Optional[str] = None,
+    nano_cpus: Optional[int] = None,
+    logger: Optional[Union[str, logging.Logger]] = None,
 ) -> Container:
     """Start a Docker container using the specified image.
 
@@ -312,19 +323,33 @@ def create_container(
 
     if not logger:
         # if logger is None, print to stdout
-        log_error = print
-        log_info = print
+        def log_error(x: str) -> None:
+            print(x)
+
+        def log_info(x: str) -> None:
+            print(x)
+
+        raise_error = True
     elif logger == "quiet":
         # if logger is "quiet", don't print anything
         def log_info(x: str) -> None:
             return None
 
         def log_error(x: str) -> None:
             return None
+
+        raise_error = True
     else:
+        assert isinstance(logger, logging.Logger)
+
         # if logger is a logger object, use it
-        log_error = logger.info
-        log_info = logger.info
+        def log_error(x: str) -> None:
+            logger.info(x)
+
+        def log_info(x: str) -> None:
+            logger.info(x)
+
+        raise_error = False
 
     container = None
     try:
@@ -349,7 +374,7 @@ def log_error(x: str) -> None:
 
 def exec_run_with_timeout(
     container: Container, cmd: str, timeout: Optional[int] = 60
-) -> None:
+) -> tuple[str, bool, float]:
     """Run a command in a container with a timeout.
 
     Args:
@@ -369,6 +394,7 @@ def exec_run_with_timeout(
     def run_command() -> None:
         nonlocal exec_result, exec_id, exception
         try:
+            assert container.client is not None, "Client did not load"
             exec_id = container.client.api.exec_create(container.id, cmd)["Id"]
             exec_stream = container.client.api.exec_start(exec_id, stream=True)
             for chunk in exec_stream:
@@ -393,3 +419,6 @@ def run_command() -> None:
         timed_out = True
     end_time = time.time()
     return exec_result, timed_out, end_time - start_time
+
+
+__all__ = []

commit0/harness/dockerfiles.py

@@ -59,9 +59,12 @@
 """
 
 
-def get_dockerfile_base(platform):
+def get_dockerfile_base(platform: str) -> str:
     return _DOCKERFILE_BASE.format(platform=platform)
 
 
-def get_dockerfile_repo(platform):
+def get_dockerfile_repo(platform: str) -> str:
     return _DOCKERFILE_REPO.format(platform=platform)
+
+
+__all__ = []

commit0/harness/run_pytest_ids.py

@@ -5,6 +5,7 @@
 import traceback
 import yaml
 from pathlib import Path
+import logging
 
 from commit0.harness.constants import RUN_PYTEST_LOG_DIR
 from commit0.harness.docker_build import (
@@ -20,7 +21,7 @@
     delete_file_from_container,
     exec_run_with_timeout,
 )
-from commit0.harness.spec import make_spec
+from commit0.harness.spec import Spec, make_spec
 from commit0.harness.utils import (
     EvaluationError,
     extract_test_output,
@@ -35,7 +36,9 @@ class ExecutionBackend(StrEnum):
     MODAL = auto()
 
 
-def run_docker(spec, logger, eval_file, timeout, log_dir):
+def run_docker(
+    spec: Spec, logger: logging.Logger, eval_file: Path, timeout: int, log_dir: Path
+) -> None:
     client = docker.from_env()
     container = None
     try:
@@ -68,7 +71,7 @@ def run_docker(spec, logger, eval_file, timeout, log_dir):
             if timed_out:
                 f.write(f"\n\nTimeout error: {timeout} seconds exceeded.")
                 raise EvaluationError(
-                    repo,
+                    spec.repo,
                     f"Test timed out after {timeout} seconds.",
                     logger,
                 )
@@ -88,7 +91,7 @@ def run_docker(spec, logger, eval_file, timeout, log_dir):
         print(e)
     except Exception as e:
         error_msg = (
-            f"Error in running pytest for {repo}: {e}\n"
+            f"Error in running pytest for {spec.repo}: {e}\n"
             f"{traceback.format_exc()}\n"
             f"Check ({logger.log_file}) for more information."
         )
@@ -99,7 +102,9 @@ def run_docker(spec, logger, eval_file, timeout, log_dir):
         close_logger(logger)
 
 
-def run_modal(spec, logger, eval_file, timeout, log_dir):
+def run_modal(
+    spec: Spec, logger: logging.Logger, eval_file: Path, timeout: int, log_dir: Path
+) -> None:
     # get image name to pull from dockerhub
     # spec.repo_image_key
     import modal
@@ -164,15 +169,15 @@ def run_modal(spec, logger, eval_file, timeout, log_dir):
         output = []
         for line in process.stderr:
             output.append(line)
-        output = "".join(line)
-        logger.info(output)
-        print(output)
+        output_s = "".join(line)
+        logger.info(output_s)
+        print(output_s)
 
         timed_out = False
         total_runtime = 1
 
         test_output = extract_test_output(
-            output, "--json-report --json-report-file=report.json"
+            output_s, "--json-report --json-report-file=report.json"
         )
 
         # stdout might be more straightforward
@@ -183,23 +188,23 @@ def run_modal(spec, logger, eval_file, timeout, log_dir):
             if timed_out:
                 f.write(f"\n\nTimeout error: {timeout} seconds exceeded.")
                 raise EvaluationError(
-                    repo,
+                    spec.repo,
                     f"Test timed out after {timeout} seconds.",
                     logger,
                 )
 
 
 def main(
     repo: str,
-    test_ids: list[str],
+    test_ids_ls: list[str],
     timeout: int,
     branch_name: str,
     backend: ExecutionBackend,
 ) -> None:
     with open("config.yml", "r") as file:
         data = yaml.safe_load(file)
     spec = make_spec(data["repos"][repo])
-    test_ids = " ".join(test_ids)
+    test_ids = " ".join(test_ids_ls)
     hashed_test_ids = get_hash_string(test_ids)
 
     # set up logging
@@ -250,7 +255,7 @@ def add_init_args(parser: argparse.ArgumentParser) -> None:
 def run(args: argparse.Namespace) -> None:
     main(
         repo=args.repo,
-        test_ids=args.test_ids,
+        test_ids_ls=args.test_ids,
         timeout=args.timeout,
         branch_name=args.branch_name,
         backend=args.backend,

commit0/harness/setup.py

@@ -5,9 +5,9 @@
 import docker
 import yaml
 from datasets import load_dataset
-
+from typing import Iterator
 from commit0.harness.utils import clone_repo
-from commit0.harness.constants import REPO_IMAGE_BUILD_DIR
+from commit0.harness.constants import REPO_IMAGE_BUILD_DIR, RepoInstance
 from commit0.harness.docker_build import build_repo_images
 from commit0.harness.spec import make_spec
 
@@ -22,7 +22,7 @@ def main(
     base_dir: str,
     config_file: str,
 ) -> None:
-    dataset = load_dataset(hf_name, split="test")
+    dataset: Iterator[RepoInstance] = load_dataset(hf_name, split="test")  # type: ignore
     out = dict()
     specs = []
     for example in dataset:
@@ -34,7 +34,9 @@ def main(
             os.path.join(base_dir, repo_name)
         )
         clone_url = f"https://github.com/{example['repo']}.git"
-        clone_repo(clone_url, out[repo_name]["local_path"], example["base_commit"])
+        clone_repo(
+            clone_url, out[repo_name]["local_path"], example["base_commit"], logger
+        )
 
     config_file = os.path.abspath(config_file)
     with open(config_file, "w") as f: