Merge pull request #34 from commit-0/integration

better integration between commit0 and baseline

Author: wenting-zhao

.github/workflows/system.yml

@@ -32,4 +32,4 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.MY_GITHUB_TOKEN }}
         run: |
-          uv run commit0 save simpy test-save-commit0
+          uv run commit0 save simpy test-save-commit0 master

baselines/class_types.py

@@ -20,8 +20,8 @@ class AgentConfig:
     max_repo_info_length: int
     use_unit_tests_info: bool
     max_unit_tests_info_length: int
-    use_reference_info: bool
-    max_reference_info_length: int
+    use_spec_info: bool
+    max_spec_info_length: int
     use_lint_info: bool
     max_lint_info_length: int
     pre_commit_config_path: str

baselines/commit0_utils.py

@@ -1,6 +1,8 @@
+import git
 import os
 import re
 import subprocess
+from dataclasses import asdict
 from pathlib import Path
 from typing import List
 
@@ -178,3 +180,65 @@ def get_reference(specification_pdf_path: str) -> str:
     """Get the reference for a given specification PDF path."""
     # TODO: after pdf_to_text is available, use it to extract the text from the PDF
     return f"/pdf {specification_pdf_path}"
+
+
+def create_branch(repo: git.Repo, branch: str, from_commit: str) -> None:
+    """Create a new branch or switch to an existing branch.
+
+    Parameters
+    ----------
+    repo : git.Repo
+        The repository object.
+    branch : str
+        The name of the branch to create or switch to.
+    from_commit : str
+        from which commit to create the branch
+
+    Returns
+    -------
+    None
+
+    Raises
+    ------
+    RuntimeError
+        If creating or switching to the branch fails.
+
+    """
+    try:
+        # Check if the branch already exists
+        if branch in repo.heads:
+            repo.git.checkout(branch)
+        else:
+            repo.git.checkout(from_commit)
+            repo.git.checkout("-b", branch)
+    except git.exc.GitCommandError as e:  # type: ignore
+        raise RuntimeError(f"Failed to create or switch to branch '{branch}': {e}")
+
+
+def args2string(agent_config: AgentConfig) -> str:
+    """Converts specific fields from an `AgentConfig` object into a formatted string.
+
+    Args:
+    ----
+        agent_config (AgentConfig): A dataclass object containing configuration
+        options for an agent.
+
+    Returns:
+    -------
+        str: A string representing the selected key-value pairs from the `AgentConfig`
+        object, joined by double underscores.
+
+    """
+    arg_dict = asdict(agent_config)
+    result_list = []
+    keys_to_collect = ["model_name", "run_tests", "use_lint_info", "use_spec_info"]
+    for key in keys_to_collect:
+        value = arg_dict[key]
+        if isinstance(value, bool):
+            if value:
+                value = "1"
+            else:
+                value = "0"
+        result_list.append(f"{key}-{value}")
+    concatenated_string = "__".join(result_list)
+    return concatenated_string

baselines/configs/agent.yaml

@@ -10,7 +10,7 @@ agent_config:
   use_user_prompt: false
   use_repo_info: false
   use_unit_tests_info: false
-  use_reference_info: false
+  use_spec_info: false
   use_lint_info: false
   pre_commit_config_path: .pre-commit-config.yaml
   run_tests: false

baselines/configs/base.yaml

@@ -17,13 +17,13 @@ agent_config:
   user_prompt: "Here is your task:\nYou need to implement all functions with 'NotImplementedError('IMPLEMENT ME HERE')' and pass the unit tests.\nDo not change the names of existing functions or classes, as they may be referenced from other code like unit tests, etc.\nWhen you generate code, you must maintain the original formatting of the function stubs (such as whitespaces), otherwise we will not able to search/replace blocks for code modifications, and therefore you will receive a score of 0 for your generated code."
   use_repo_info: false
   use_unit_tests_info: false
-  use_reference_info: false
+  use_spec_info: false
   use_lint_info: false
   pre_commit_config_path: .pre-commit-config.yaml
   run_tests: True
   max_repo_info_length: 10000
   max_unit_tests_info_length: 10000
-  max_reference_info_length: 10000
+  max_spec_info_length: 10000
   max_lint_info_length: 10000
   max_iteration: 3
 

baselines/run_agent.py

@@ -1,9 +1,12 @@
 import os
 import sys
 import hydra
-from datasets import load_dataset
 import traceback
+from datasets import load_dataset
+from git import Repo
 from baselines.commit0_utils import (
+    args2string,
+    create_branch,
     get_message,
     get_target_edit_files,
 )
@@ -54,6 +57,12 @@ def run_agent_for_repo(
 
     repo_path = os.path.join(commit0_config.base_dir, repo_name)
     repo_path = os.path.abspath(repo_path)
+    try:
+        local_repo = Repo(repo_path)
+    except Exception:
+        raise Exception(
+            f"{repo_path} is not a git repo. Check if base_dir is correctly specified."
+        )
 
     target_edit_files = get_target_edit_files(repo_path)
 
@@ -64,6 +73,16 @@ def run_agent_for_repo(
             f"{agent_config.agent_name} is not implemented; please add your implementations in baselines/agents.py."
         )
 
+    run_id = args2string(agent_config)
+    print(f"Agent is coding on branch: {run_id}", file=sys.stderr)
+    create_branch(local_repo, run_id, example["base_commit"])
+    latest_commit = local_repo.commit(run_id)
+    # in cases where the latest commit of branch is not commit 0
+    # set it back to commit 0
+    # TODO: ask user for permission
+    if latest_commit.hexsha != example["base_commit"]:
+        local_repo.git.reset("--hard", example["base_commit"])
+
     with DirContext(repo_path):
         if commit0_config is None or agent_config is None:
             raise ValueError("Invalid input")
@@ -78,7 +97,7 @@ def run_agent_for_repo(
         if agent_config.run_tests:
             # when unit test feedback is available, iterate over test files
             for test_file in test_files:
-                test_cmd = f"python -m commit0 test {repo_path} {test_file}"
+                test_cmd = f"python -m commit0 test {repo_path} {run_id} {test_file}"
                 test_file_name = test_file.replace(".py", "").replace("/", "__")
                 log_dir = RUN_AIDER_LOG_DIR / "with_tests" / test_file_name
 

commit0/__main__.py

@@ -46,14 +46,21 @@ def main() -> None:
     config.base_dir = os.path.abspath(config.base_dir)
 
     if command == "clone":
+        if len(sys.argv) != 3:
+            raise ValueError(
+                "You provided an incorrect number of arguments.\nUsage: commit0 clone {repo_split}"
+            )
         commit0.harness.setup.main(
             config.dataset_name,
             config.dataset_split,
             config.repo_split,
             config.base_dir,
-            config.branch,
         )
     elif command == "build":
+        if len(sys.argv) != 3:
+            raise ValueError(
+                "You provided an incorrect number of arguments.\nUsage: commit0 build {repo_split}"
+            )
         commit0.harness.build.main(
             config.dataset_name,
             config.dataset_split,
@@ -62,20 +69,37 @@ def main() -> None:
             config.backend,
         )
     elif command == "get-tests":
+        if len(sys.argv) != 3:
+            raise ValueError(
+                "You provided an incorrect number of arguments.\nUsage: commit0 get-tests {repo_name}"
+            )
         repo = sys.argv[2]
         commit0.harness.get_pytest_ids.main(repo, stdout=True)
     elif command == "test" or command == "test-reference":
         # this command assume execution in arbitrary working directory
         repo_or_repo_path = sys.argv[2]
-        test_ids = sys.argv[3]
         if command == "test-reference":
-            config.branch = "reference"
+            if len(sys.argv) != 4:
+                raise ValueError(
+                    "You provided an incorrect number of arguments.\nUsage: commit0 test-reference {repo_dir} {test_ids}"
+                )
+            branch = "reference"
+            test_ids = sys.argv[3]
+        else:
+            if len(sys.argv) != 5:
+                raise ValueError(
+                    "You provided an incorrect number of arguments.\nUsage: commit0 test {repo_dir} {branch} {test_ids}"
+                )
+            branch = sys.argv[3]
+            test_ids = sys.argv[4]
+        if branch.startswith("branch="):
+            branch = branch[len("branch=") :]
         commit0.harness.run_pytest_ids.main(
             config.dataset_name,
             config.dataset_split,
             config.base_dir,
             repo_or_repo_path,
-            config.branch,
+            branch,
             test_ids,
             config.backend,
             config.timeout,
@@ -84,27 +108,46 @@ def main() -> None:
         )
     elif command == "evaluate" or command == "evaluate-reference":
         if command == "evaluate-reference":
-            config.branch = "reference"
+            if len(sys.argv) != 3:
+                raise ValueError(
+                    "You provided an incorrect number of arguments.\nUsage: commit0 evaluate-reference {repo_split}"
+                )
+            branch = "reference"
+        else:
+            if len(sys.argv) != 4:
+                raise ValueError(
+                    "You provided an incorrect number of arguments.\nUsage: commit0 evaluate {repo_split} {branch}"
+                )
+            branch = sys.argv[3]
+        if branch.startswith("branch="):
+            branch = branch[len("branch=") :]
         commit0.harness.evaluate.main(
             config.dataset_name,
             config.dataset_split,
             config.repo_split,
             config.base_dir,
-            config.branch,
+            branch,
             config.backend,
             config.timeout,
             config.num_cpus,
             config.num_workers,
         )
     elif command == "save":
-        organization = sys.argv[3]
+        if len(sys.argv) != 5:
+            raise ValueError(
+                "You provided an incorrect number of arguments.\nUsage: commit0 save {repo_split} {owner} {branch}"
+            )
+        owner = sys.argv[3]
+        branch = sys.argv[4]
+        if branch.startswith("branch="):
+            branch = branch[len("branch=") :]
         commit0.harness.save.main(
             config.dataset_name,
             config.dataset_split,
             config.repo_split,
             config.base_dir,
-            organization,
-            config.branch,
+            owner,
+            branch,
             config.github_token,
         )
 

commit0/configs/base.yaml

@@ -14,7 +14,6 @@ num_workers: 8
 
 # test related
 backend: local
-branch: ai
 timeout: 1_800
 num_cpus: 1
 

commit0/configs/config_class.py

@@ -18,8 +18,6 @@ class Commit0Config:
 
     # test related
     backend: str
-    # which branch to work on
-    branch: str
     # timeout for running pytest
     timeout: int
     num_cpus: int

commit0/harness/run_pytest_ids.py

@@ -77,13 +77,15 @@ def main(
             local_repo = git.Repo(repo_dir)
         except git.exc.NoSuchPathError:  # type: ignore
             raise Exception(
-                f"{repo_dir} and {repo_or_repo_dir} are not git directories.\nUsage: commit0 test {{repo_dir}} {test_ids}"
+                f"{repo_dir} and {repo_or_repo_dir} are not git directories.\nUsage: commit0 test {{repo_dir}} {{branch}} {{test_ids}}"
             )
         except Exception as e:
             raise e
     if branch == "reference":
         commit_id = example["reference_commit"]
     else:
+        if branch not in local_repo.branches:
+            raise Exception(f"Branch {branch} does not exist.")
         local_branch = local_repo.branches[branch]
         commit_id = local_branch.commit.hexsha
     patch = generate_patch_between_commits(

commit0/harness/save.py

@@ -20,7 +20,7 @@ def main(
     dataset_split: str,
     repo_split: str,
     base_dir: str,
-    organization: str,
+    owner: str,
     branch: str,
     github_token: str,
 ) -> None:
@@ -33,7 +33,7 @@ def main(
         if repo_split != "all" and repo_name not in SPLIT[repo_split]:
             continue
         local_repo_path = f"{base_dir}/{repo_name}"
-        github_repo_url = f"https://github.com/{organization}/{repo_name}.git"
+        github_repo_url = f"https://github.com/{owner}/{repo_name}.git"
         github_repo_url = github_repo_url.replace(
             "https://", f"https://x-access-token:{github_token}@"
         )
@@ -46,7 +46,7 @@ def main(
 
         # create Github repo
         create_repo_on_github(
-            organization=organization, repo=repo_name, logger=logger, token=github_token
+            organization=owner, repo=repo_name, logger=logger, token=github_token
         )
         # Add your remote repository URL
         remote_name = "progress-tracker"
@@ -75,9 +75,7 @@ def main(
             origin.push(refspec=f"{branch}:{branch}")
             logger.info(f"Pushed to {github_repo_url} on branch {branch}")
         except Exception as e:
-            raise Exception(
-                f"Push {branch} to {organization}/{repo_name} fails.\n{str(e)}"
-            )
+            raise Exception(f"Push {branch} to {owner}/{repo_name} fails.\n{str(e)}")
 
 
 __all__ = []

commit0/harness/setup.py

@@ -6,7 +6,6 @@
 from typing import Iterator
 from commit0.harness.utils import (
     clone_repo,
-    create_branch,
 )
 from commit0.harness.constants import RepoInstance, SPLIT
 
@@ -18,7 +17,10 @@
 
 
 def main(
-    dataset_name: str, dataset_split: str, repo_split: str, base_dir: str, branch: str
+    dataset_name: str,
+    dataset_split: str,
+    repo_split: str,
+    base_dir: str,
 ) -> None:
     dataset: Iterator[RepoInstance] = load_dataset(dataset_name, split=dataset_split)  # type: ignore
     for example in dataset:
@@ -27,8 +29,7 @@ def main(
             continue
         clone_url = f"https://github.com/{example['repo']}.git"
         clone_dir = os.path.abspath(os.path.join(base_dir, repo_name))
-        local_repo = clone_repo(clone_url, clone_dir, example["base_commit"], logger)
-        create_branch(local_repo, branch, logger)
+        clone_repo(clone_url, clone_dir, example["base_commit"], logger)
 
 
 __all__ = []