Skip to content

Commit 1439727

Browse files
nanjiangwillnanjiangwill
committed
update
1 parent ae24883 commit 1439727

File tree

3 files changed

+35
-18
lines changed

3 files changed

+35
-18
lines changed

‎agent/README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ Available options include:
3838
`--max-lint-info-length: int`: Maximum length of the lint information to use. [Default: `10000`]
3939
`--pre-commit-config-path: str`: Path to the pre-commit config file. This is needed for running `lint`. [Default: `.pre-commit-config.yaml`]
4040
`--agent-config-file: str`: Path to write the agent config. [Default: `.agent.yaml`]
41+
`--add-import-module-to-context: bool`: Add import module to context. [Default: `False`]
42+
`--record-test-for-each-commit: bool`: Record test results for each commit. [Default: `False`], if set to `True`, the test results will be saved in `experiment_log_dir/eval_results.json`
4143

4244
## Running Agent
4345
Use `agent run [OPTIONS] BRANCH` to execute an agent on a specific branch.

‎agent/run_agent.py

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
get_lint_cmd,
1313
read_yaml_config,
1414
)
15+
import json
1516
import subprocess
1617
from agent.agents import AiderAgents
1718
from typing import Optional, Type, cast
@@ -46,20 +47,18 @@ def __exit__(
4647

4748

4849
def run_eval_after_each_commit(
49-
branch: str, backend: str, commit0_config_file: str, file_log_dir: Path
50-
) -> None:
50+
branch: str, backend: str, commit0_config_file: str
51+
) -> str:
5152
"""Run the eval command after each commit."""
5253
eval_cmd = f"python -m commit0 evaluate --branch {branch} --backend {backend} --commit0-config-file {commit0_config_file} --timeout 100"
5354
try:
5455
result = subprocess.run(
5556
eval_cmd, shell=True, capture_output=True, text=True, check=True
5657
)
57-
with open(file_log_dir / "current_commit_eval_result.txt", "w") as f:
58-
f.write(result.stdout)
58+
return result.stdout
5959
except subprocess.CalledProcessError as e:
6060
print(f"Error running eval command: {e}")
61-
with open(file_log_dir / "current_commit_eval_result.txt", "w") as f:
62-
f.write(e.stdout if e.stdout else str(e))
61+
return e.stdout if e.stdout else str(e)
6362

6463

6564
def run_agent_for_repo(
@@ -147,6 +146,7 @@ def run_agent_for_repo(
147146
)
148147
experiment_log_dir.mkdir(parents=True, exist_ok=True)
149148

149+
eval_results = {}
150150
# write agent_config to .agent.yaml in the log_dir for record
151151
agent_config_log_file = experiment_log_dir / ".agent.yaml"
152152
with open(agent_config_log_file, "w") as agent_config_file:
@@ -179,8 +179,9 @@ def run_agent_for_repo(
179179
test_first=True,
180180
)
181181
if agent_config.record_test_for_each_commit:
182-
run_eval_after_each_commit(
183-
branch, backend, commit0_config_file, test_log_dir
182+
current_commit = local_repo.head.commit.hexsha
183+
eval_results[current_commit] = run_eval_after_each_commit(
184+
branch, backend, commit0_config_file
184185
)
185186

186187
# after running the agent, update the money display
@@ -211,8 +212,9 @@ def run_agent_for_repo(
211212
lint_first=True,
212213
)
213214
if agent_config.record_test_for_each_commit:
214-
run_eval_after_each_commit(
215-
branch, backend, commit0_config_file, lint_log_dir
215+
current_commit = local_repo.head.commit.hexsha
216+
eval_results[current_commit] = run_eval_after_each_commit(
217+
branch, backend, commit0_config_file
216218
)
217219

218220
# after running the agent, update the money display
@@ -239,8 +241,9 @@ def run_agent_for_repo(
239241
)
240242
agent_return = agent.run(message, "", lint_cmd, [f], file_log_dir)
241243
if agent_config.record_test_for_each_commit:
242-
run_eval_after_each_commit(
243-
branch, backend, commit0_config_file, file_log_dir
244+
current_commit = local_repo.head.commit.hexsha
245+
eval_results[current_commit] = run_eval_after_each_commit(
246+
branch, backend, commit0_config_file
244247
)
245248

246249
update_queue.put(
@@ -249,6 +252,10 @@ def run_agent_for_repo(
249252
(repo_name, file_name, agent_return.last_cost),
250253
)
251254
)
255+
if agent_config.record_test_for_each_commit:
256+
with open(experiment_log_dir / "eval_results.json", "w") as f:
257+
json.dump(eval_results, f)
258+
252259
update_queue.put(("finish_repo", repo_name))
253260

254261

‎agent/run_agent_no_rich.py

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
read_yaml_config,
1515
)
1616
import subprocess
17+
import json
1718
from agent.agents import AiderAgents
1819
from typing import cast
1920
from agent.class_types import AgentConfig
@@ -106,6 +107,7 @@ def run_agent_for_repo(
106107
/ datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
107108
)
108109
experiment_log_dir.mkdir(parents=True, exist_ok=True)
110+
eval_results = {}
109111

110112
# write agent_config to .agent.yaml in the log_dir for record
111113
agent_config_log_file = experiment_log_dir / ".agent.yaml"
@@ -137,8 +139,9 @@ def run_agent_for_repo(
137139
test_first=True,
138140
)
139141
if agent_config.record_test_for_each_commit:
140-
run_eval_after_each_commit(
141-
branch, backend, commit0_config_file, test_log_dir
142+
current_commit = local_repo.head.commit.hexsha
143+
eval_results[current_commit] = run_eval_after_each_commit(
144+
branch, backend, commit0_config_file
142145
)
143146
elif agent_config.run_entire_dir_lint:
144147
# when unit test feedback is available, iterate over test files
@@ -159,8 +162,9 @@ def run_agent_for_repo(
159162
lint_first=True,
160163
)
161164
if agent_config.record_test_for_each_commit:
162-
run_eval_after_each_commit(
163-
branch, backend, commit0_config_file, lint_log_dir
165+
current_commit = local_repo.head.commit.hexsha
166+
eval_results[current_commit] = run_eval_after_each_commit(
167+
branch, backend, commit0_config_file
164168
)
165169
else:
166170
# when unit test feedback is not available, iterate over target files to edit
@@ -177,9 +181,13 @@ def run_agent_for_repo(
177181
)
178182
_ = agent.run(message, "", lint_cmd, [f], file_log_dir)
179183
if agent_config.record_test_for_each_commit:
180-
run_eval_after_each_commit(
181-
branch, backend, commit0_config_file, file_log_dir
184+
current_commit = local_repo.head.commit.hexsha
185+
eval_results[current_commit] = run_eval_after_each_commit(
186+
branch, backend, commit0_config_file
182187
)
188+
if agent_config.record_test_for_each_commit:
189+
with open(experiment_log_dir / "eval_results.json", "w") as f:
190+
json.dump(eval_results, f)
183191

184192

185193
def run_agent(

0 commit comments

Comments
 (0)