adding logging

Author: nanjiangwill

baselines/class_types.py

@@ -24,3 +24,4 @@ class AiderConfig:
     use_lint_info: bool
     max_lint_info_length: int
     pre_commit_config_path: str
+    run_tests: bool

baselines/configs/aider.yaml

@@ -9,4 +9,5 @@ aider_config:
   use_unit_tests_info: false
   use_reference_info: false
   use_lint_info: true
-  pre_commit_config_path: .pre-commit-config.yaml
+  pre_commit_config_path: .pre-commit-config.yaml
+  run_tests: true

baselines/configs/base.yaml

@@ -7,7 +7,7 @@ commit0_config:
   base_dir: /Users/willjiang/Desktop/ai2dev/commit0/repos
   dataset_name: "wentingzhao/commit0_docstring"
   dataset_split: "test"
-  repo_split: "lite"
+  repo_split: "simpy"
   num_workers: 10
 
 aider_config:
@@ -23,6 +23,7 @@ aider_config:
   use_lint_info: false
   max_lint_info_length: 10000
   pre_commit_config_path: .pre-commit-config.yaml
+  run_tests: True
 
 hydra:
   run:

baselines/run_aider.py

@@ -1,7 +1,7 @@
 import logging
 import os
 import subprocess
-
+from pathlib import Path
 import hydra
 from datasets import load_dataset
 import traceback
@@ -15,12 +15,8 @@
 from commit0.harness.get_pytest_ids import main as get_tests
 from tqdm import tqdm
 from concurrent.futures import ThreadPoolExecutor, as_completed
-
-
-logging.basicConfig(
-    level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-)
-logger = logging.getLogger(__name__)
+from commit0.harness.constants import RUN_AIDER_LOG_DIR
+from commit0.harness.docker_build import setup_logger
 
 
 def get_aider_cmd(
@@ -29,6 +25,7 @@ def get_aider_cmd(
     message_to_aider: str,
     test_cmd: str,
     lint_cmd: str,
+    log_dir: Path,
 ) -> str:
     """Get the Aider command based on the given context."""
     base_cmd = f'aider --model {model} --file {files} --message "{message_to_aider}"'
@@ -37,9 +34,45 @@ def get_aider_cmd(
     if test_cmd:
         base_cmd += f" --auto-test --test --test-cmd '{test_cmd}'"
     base_cmd += " --yes"
+
+    # Store Aider input and chat history in log directory
+    input_history_file = log_dir / ".aider.input.history"
+    chat_history_file = log_dir / ".aider.chat.history.md"
+
+    base_cmd += f" --input-history-file {input_history_file}"
+    base_cmd += f" --chat-history-file {chat_history_file}"
     return base_cmd
 
 
+def execute_aider_cmd(
+    aider_cmd: str,
+    logger: logging.Logger,
+) -> None:
+    """Execute the Aider command."""
+    try:
+        process = subprocess.Popen(
+            aider_cmd,
+            shell=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            universal_newlines=True,
+        )
+        stdout, stderr = process.communicate()
+        logger.info(f"STDOUT: {stdout}")
+        logger.info(f"STDERR: {stderr}")
+    except subprocess.CalledProcessError as e:
+        logger.error(f"Command failed with exit code {e.returncode}")
+        logger.error(f"STDOUT: {e.stdout}")
+        logger.error(f"STDERR: {e.stderr}")
+
+    except OSError as e:
+        if e.errno == 63:  # File name too long error
+            logger.error("Command failed due to file name being too long")
+            logger.error(f"Command: {''.join(aider_cmd)}")
+        else:
+            logger.error(f"OSError occurred: {e}")
+
+
 def run_aider_for_repo(
     commit0_config: Commit0Config | None,
     aider_config: AiderConfig | None,
@@ -56,7 +89,7 @@ def run_aider_for_repo(
     repo_name = repo_name.replace(".", "-")
 
     # Call the commit0 get-tests command to retrieve test files
-    test_files_str = get_tests(repo_name, stdout=True)
+    test_files_str = get_tests(repo_name, stdout=False)
 
     test_files = sorted(list(set([i.split(":")[0] for i in test_files_str])))
 
@@ -75,39 +108,55 @@ def run_aider_for_repo(
     else:
         lint_cmd = ""
 
-    for test_file in test_files:
-        test_cmd = f"python -m commit0 test {repo_name} {test_file}"
+    if aider_config.run_tests:
+        for test_file in test_files:
+            test_cmd = f"python -m commit0 test {repo_name} {test_file}"
+            # set up logging
+            test_file_name = test_file.replace(".py", "").replace("/", "__")
+            log_dir = RUN_AIDER_LOG_DIR / repo_name / "ai" / test_file_name
+            log_dir.mkdir(parents=True, exist_ok=True)
+            log_file = log_dir / "run_aider.log"
+            logger = setup_logger(repo_name, log_file)
+
+            aider_cmd = get_aider_cmd(
+                aider_config.llm_name,
+                target_edit_files_cmd_args,
+                message_to_aider,
+                test_cmd,
+                lint_cmd,
+                log_dir,
+            )
+
+            # write aider command to log file
+            aider_cmd_file = Path(log_dir / "aider_cmd.sh")
+            aider_cmd_file.write_text(aider_cmd)
+
+            # write test command to log file
+            test_cmd_file = Path(log_dir / "test_cmd.sh")
+            test_cmd_file.write_text(test_cmd)
+
+            execute_aider_cmd(aider_cmd, logger)
+
+    else:
+        # set up logging
+        log_dir = RUN_AIDER_LOG_DIR / repo_name / "ai" / "no_test"
+        log_dir.mkdir(parents=True, exist_ok=True)
+        log_file = log_dir / "run_aider.log"
+        logger = setup_logger(repo_name, log_file)
 
         aider_cmd = get_aider_cmd(
             aider_config.llm_name,
             target_edit_files_cmd_args,
             message_to_aider,
-            test_cmd,
+            "",
             lint_cmd,
+            log_dir,
         )
+        # write aider command to log file
+        aider_cmd_file = Path(log_dir / "aider_cmd.sh")
+        aider_cmd_file.write_text(aider_cmd)
 
-        try:
-            process = subprocess.Popen(
-                aider_cmd,
-                shell=True,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
-                universal_newlines=True,
-            )
-            stdout, stderr = process.communicate()
-            logger.info(f"STDOUT: {stdout}")
-            logger.info(f"STDERR: {stderr}")
-        except subprocess.CalledProcessError as e:
-            logger.error(f"Command failed with exit code {e.returncode}")
-            logger.error(f"STDOUT: {e.stdout}")
-            logger.error(f"STDERR: {e.stderr}")
-
-        except OSError as e:
-            if e.errno == 63:  # File name too long error
-                logger.error("Command failed due to file name being too long")
-                logger.error(f"Command: {''.join(aider_cmd)}")
-            else:
-                logger.error(f"OSError occurred: {e}")
+        execute_aider_cmd(aider_cmd, logger)
 
 
 def pre_aider_processing(aider_config: AiderConfig) -> None:
@@ -150,7 +199,7 @@ def main() -> None:
             and example["repo"].split("/")[-1]
             in SPLIT.get(commit0_config.repo_split, [])
         )
-    ][:1]
+    ]
 
     pre_aider_processing(aider_config)
 

commit0/harness/constants.py

@@ -20,6 +20,7 @@ class Files(TypedDict):
 BASE_IMAGE_BUILD_DIR = Path("logs/build_images/base")
 REPO_IMAGE_BUILD_DIR = Path("logs/build_images/repo")
 RUN_PYTEST_LOG_DIR = Path("logs/pytest")
+RUN_AIDER_LOG_DIR = Path("logs/aider")
 
 # Constants - Test Types, Statuses, Commands
 FAIL_TO_PASS = "FAIL_TO_PASS"