Merge pull request #27 from commit-0/patch

Patch

Author: wenting-zhao

.github/workflows/system.yml

@@ -22,8 +22,6 @@ jobs:
         run: uv run commit0 clone simpy
       - name: Build docker images
         run: uv run commit0 build simpy
-      - name: Set up git user
-        run: sudo "$(which uv)" run commit0 setup-git-user simpy
       - name: Get tests
         run: uv run commit0 get-tests simpy
       - name: Test

commit0/__main__.py

@@ -2,7 +2,6 @@
 import commit0.harness.get_pytest_ids
 import commit0.harness.build
 import commit0.harness.setup
-import commit0.harness.setup_git_user
 import commit0.harness.evaluate
 import commit0.harness.save
 import copy
@@ -34,7 +33,6 @@ def main() -> None:
     if command in [
         "clone",
         "build",
-        "setup-git-user",
         "evaluate",
         "evaluate-reference",
         "save",
@@ -62,16 +60,6 @@ def main() -> None:
             config.repo_split,
             config.num_workers,
             config.backend,
-            config.key_path,
-        )
-    elif command == "setup-git-user":
-        commit0.harness.setup_git_user.main(
-            config.dataset_name,
-            config.dataset_split,
-            config.repo_split,
-            config.base_dir,
-            config.git_user,
-            config.key_path,
         )
     elif command == "get-tests":
         repo = sys.argv[2]

commit0/configs/base.yaml

@@ -11,10 +11,6 @@ repo_split: all
 
 # build related
 num_workers: 8
-key_path: commit0/configs/public_keys.json
-
-# set up git user
-git_user: git  # by default, git user is called git
 
 # test related
 backend: local

commit0/configs/config_class.py

@@ -15,8 +15,6 @@ class Commit0Config:
     # build related
     # which repo to build, all or one repo
     num_workers: int
-    # path to store public keys from docker images
-    key_path: str
 
     # test related
     backend: str

commit0/harness/build.py

@@ -1,19 +1,11 @@
-import json
 import logging
-import traceback
 
 import docker
 from datasets import load_dataset
-from tqdm import tqdm
 from typing import Iterator
 
-from commit0.harness.constants import EVAL_BACKENDS, RepoInstance, SPLIT
+from commit0.harness.constants import RepoInstance, SPLIT
 from commit0.harness.docker_build import build_repo_images
-from commit0.harness.execution_context import (
-    ExecutionBackend,
-    Docker,
-    Modal,
-)
 from commit0.harness.spec import make_spec
 
 logging.basicConfig(
@@ -28,7 +20,6 @@ def main(
     repo_split: str,
     num_workers: int,
     backend: str,
-    key_path: str,
 ) -> None:
     dataset: Iterator[RepoInstance] = load_dataset(dataset_name, split=dataset_split)  # type: ignore
     specs = []
@@ -39,29 +30,9 @@ def main(
         spec = make_spec(example)
         specs.append(spec)
 
-    if ExecutionBackend(backend) == ExecutionBackend.MODAL:
-        execution_context = Modal
-    elif ExecutionBackend(backend) == ExecutionBackend.LOCAL:
+    if backend == "local":
         client = docker.from_env()
         build_repo_images(client, specs, num_workers)
-        execution_context = Docker
-    else:
-        raise ValueError(
-            f"Evaluation must be from {', '.join(EVAL_BACKENDS)}, but {backend} is provided."
-        )
-
-    # get ssh key from each docker image
-    img2key = dict()
-    for spec in tqdm(specs, desc="Retrieving public keys from docker images"):
-        try:
-            with execution_context(spec, logger, timeout=60) as context:
-                key = context.get_ssh_pubkey_from_remote(user="root")
-                img2key[spec.repo_image_key] = key
-        except Exception as e:
-            error_msg = f"General error: {e}\n" f"{traceback.format_exc()}\n"
-            raise RuntimeError(error_msg)
-    with open(key_path, "w") as json_file:
-        json.dump(img2key, json_file, indent=4)
 
 
 __all__ = []

commit0/harness/constants.py

@@ -13,6 +13,7 @@ class RepoInstance(TypedDict):
 
 class Files(TypedDict):
     eval_script: Dict[str, Path]
+    patch: Dict[str, Path]
 
 
 # Constants - Evaluation Log Directories
@@ -33,7 +34,6 @@ class Files(TypedDict):
 COMMANDS = [
     "clone",
     "build",
-    "setup-git-user",
     "test",
     "test-reference",
     "get-tests",

commit0/harness/docker_utils.py

@@ -140,33 +140,6 @@ def delete_file_from_container(container: Container, file_path: str) -> None:
         raise Exception(f"General Error: {str(e)}")
 
 
-def get_ssh_pubkey_from_container(container: Container, user: str) -> str:
-    """Copy the SSH public key from a Docker container to the local authorized_keys file.
-
-    Args:
-    ----
-    container (Container): Docker container to copy the key from.
-    user (str): to get public key of which user
-
-    Returns:
-    -------
-    public_key (str): public key from docker container
-
-    Raises:
-    ------
-    docker.errors.APIError: If there is an error calling the Docker API.
-
-    """
-    try:
-        exit_code, output = container.exec_run("cat /root/.ssh/id_rsa.pub")
-        if exit_code != 0:
-            raise Exception(f"Error reading file: {output.decode('utf-8').strip()}")
-        public_key = output.decode("utf-8").strip()
-        return public_key
-    except docker.errors.APIError as e:
-        raise docker.errors.APIError(f"Docker API Error: {str(e)}")
-
-
 def write_to_container(container: Container, data: str, dst: Path) -> None:
     """Write a string to a file in a docker container"""
     # echo with heredoc to file

commit0/harness/dockerfiles.py

@@ -7,7 +7,6 @@
 
 RUN apt update && apt install -y \
 wget \
-git \
 build-essential \
 libffi-dev \
 libtiff-dev \
@@ -21,16 +20,10 @@
 tzdata \
 && rm -rf /var/lib/apt/lists/*
 
-# Define arguments for SSH key parameters
-ARG SSH_KEY_PATH="/root/.ssh"
-ARG SSH_KEY_NAME="id_rsa"
-ARG SSH_KEY_PASSPHRASE=""
-
-# Create the .ssh directory
-RUN mkdir -p ${{SSH_KEY_PATH}}
-
-# Generate SSH keys
-RUN ssh-keygen -t rsa -b 4096 -f ${{SSH_KEY_PATH}}/${{SSH_KEY_NAME}} -N "${{SSH_KEY_PASSPHRASE}}"
+# Install the latest version of Git
+RUN apt-get update && apt-get install software-properties-common -y
+RUN add-apt-repository ppa:git-core/ppa -y
+RUN apt-get update && apt-get install git -y
 
 # Set up uv
 # The installer requires curl (and certificates) to download the release archive

commit0/harness/execution_context.py

@@ -27,7 +27,6 @@
     create_container,
     copy_from_container,
     copy_to_container,
-    get_ssh_pubkey_from_container,
     delete_file_from_container,
     exec_run_with_timeout,
 )
@@ -62,11 +61,6 @@ def __init__(
         self.logger = logger
         self.timeout = timeout
 
-    @abstractmethod
-    def get_ssh_pubkey_from_remote(self) -> None:
-        """Copy"""
-        raise NotImplementedError
-
     @abstractmethod
     def exec_run_with_timeout(
         self, command: str, timeout: int
@@ -148,10 +142,6 @@ def __init__(
             for _, f in files_to_copy.items():
                 copy_to_container(self.container, f["src"], f["dest"])  # type: ignore
 
-    def get_ssh_pubkey_from_remote(self, user: str) -> str:
-        """Copy"""
-        return get_ssh_pubkey_from_container(self.container, user)
-
     def exec_run_with_timeout(
         self, command: str, timeout: int
     ) -> tuple[str, bool, float]:
@@ -206,12 +196,6 @@ def __init__(
             timeout=timeout,
         )
 
-    def get_ssh_pubkey_from_remote(self, user: str) -> str:
-        """Copy ssh pubkey"""
-        process = self.sandbox.exec("bash", "-c", f"cat /{user}/.ssh/id_rsa.pub")
-        public_key = "".join([line for line in process.stdout]).strip()
-        return public_key
-
     def exec_run_with_timeout(
         self, command: str, timeout: int
     ) -> tuple[str, bool, float]:

commit0/harness/run_pytest_ids.py

@@ -18,7 +18,7 @@
     EvaluationError,
     extract_test_output,
     get_hash_string,
-    get_ip,
+    generate_patch_between_commits,
 )
 from commit0.harness.execution_context import (
     ExecutionBackend,
@@ -60,20 +60,20 @@ def main(
     log_file = log_dir / "run_pytest.log"
     logger = setup_logger(repo, log_file)
 
+    local_repo = Repo(f"{base_dir}/{repo}")
     if branch == "reference":
         commit_id = example["reference_commit"]
     else:
-        local_repo = Repo(f"{base_dir}/{repo}")
         local_branch = local_repo.branches[branch]
         commit_id = local_branch.commit.hexsha
+    patch = generate_patch_between_commits(
+        local_repo, example["base_commit"], commit_id
+    )
+    patch_file = Path(log_dir / "patch.diff")
+    patch_file.write_text(patch)
 
     # make eval file
-    eval_script = spec.eval_script.format(
-        local_repo=f"{base_dir}/{repo}",
-        commit_id=commit_id,
-        test_ids=test_ids,
-        ip=get_ip(backend),
-    )
+    eval_script = spec.eval_script.format(test_ids=test_ids)
     eval_file = Path(log_dir / "eval.sh")
     eval_file.write_text(eval_script)
 
@@ -86,7 +86,10 @@ def main(
             f"Evaluation must be from {', '.join(EVAL_BACKENDS)}, but {backend} is provided."
         )
 
-    files_to_copy = Files(eval_script={"src": eval_file, "dest": Path("/eval.sh")})
+    files_to_copy = Files(
+        eval_script={"src": eval_file, "dest": Path("/eval.sh")},
+        patch={"src": patch_file, "dest": Path("/patch.diff")},
+    )
 
     try:
         with execution_context(spec, logger, timeout, files_to_copy) as context:

commit0/harness/setup_git_user.py

@@ -147,18 +147,14 @@ def make_repo_script_list(instance: RepoInstance, repo_directory: str) -> list[s
 
 def make_eval_script_list(instance: RepoInstance, repo_directory: str) -> list[str]:
     """Run the tests."""
-    origin_name = "tmp-test"
     eval_script_list = [
-        "ssh-keyscan {ip} >> ~/.ssh/known_hosts",
         f"cd {repo_directory}",
         "source .venv/bin/activate",
-        f"git remote add {origin_name} ssh://git@{{ip}}:{{local_repo}}",
-        f"git fetch {origin_name}",
-        "git checkout {commit_id}",
+        f"git reset --hard {instance['base_commit']}",
+        "git apply --allow-empty -v /patch.diff",
         "git status",
         f"{instance['test']['test_cmd']} --json-report --json-report-file=report.json {{test_ids}}",
-        f"git checkout {instance['base_commit']}",
-        f"git remote remove {origin_name}",
+        f"git reset --hard {instance['base_commit']}",
         "git status",
     ]
     return eval_script_list