Skip to content

Commit 88d8526

Browse files
authored
Merge pull request #29 from commit-0/modal-fix
Get modal working + simplify ExecutionContext class
2 parents c53f936 + c9a3973 commit 88d8526

File tree

8 files changed

+92
-122
lines changed

8 files changed

+92
-122
lines changed

commit0/__main__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ def main() -> None:
7878
test_ids,
7979
config.backend,
8080
config.timeout,
81+
config.num_cpus,
8182
stdout=True,
8283
)
8384
elif command == "evaluate" or command == "evaluate-reference":
@@ -91,6 +92,7 @@ def main() -> None:
9192
config.branch,
9293
config.backend,
9394
config.timeout,
95+
config.num_cpus,
9496
config.num_workers,
9597
)
9698
elif command == "save":

commit0/configs/base.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ num_workers: 8
1616
backend: local
1717
branch: ai
1818
timeout: 1_800
19+
num_cpus: 1
1920

2021
# save related
2122
github_token: null

commit0/configs/config_class.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ class Commit0Config:
2222
branch: str
2323
# timeout for running pytest
2424
timeout: int
25+
num_cpus: int
2526

2627
# save related
2728
github_token: Optional[str]

commit0/harness/docker_utils.py

Lines changed: 2 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -116,30 +116,6 @@ def safe_extract(
116116
extracted_file_path.rename(dst)
117117

118118

119-
def delete_file_from_container(container: Container, file_path: str) -> None:
120-
"""Delete a file from a docker container.
121-
122-
Args:
123-
----
124-
container (Container): Docker container to delete the file from
125-
file_path (str): Path to the file in the container to be deleted
126-
127-
Raises:
128-
------
129-
docker.errors.APIError: If there is an error calling the Docker API.
130-
Exception: If the file deletion command fails with a non-zero exit code.
131-
132-
"""
133-
try:
134-
exit_code, output = container.exec_run(f"rm -f {file_path}")
135-
if exit_code != 0:
136-
raise Exception(f"Error deleting file: {output.decode('utf-8').strip()}")
137-
except docker.errors.APIError as e:
138-
raise docker.errors.APIError(f"Docker API Error: {str(e)}")
139-
except Exception as e:
140-
raise Exception(f"General Error: {str(e)}")
141-
142-
143119
def write_to_container(container: Container, data: str, dst: Path) -> None:
144120
"""Write a string to a file in a docker container"""
145121
# echo with heredoc to file
@@ -247,7 +223,7 @@ def create_container(
247223
image_name: str,
248224
container_name: Optional[str] = None,
249225
user: Optional[str] = None,
250-
command: Optional[str] = None,
226+
command: Optional[str] = "tail -f /dev/null",
251227
nano_cpus: Optional[int] = None,
252228
logger: Optional[Union[str, logging.Logger]] = None,
253229
) -> Container:
@@ -312,7 +288,7 @@ def log_info(x: str) -> None:
312288
image=image_name,
313289
name=container_name,
314290
user=user,
315-
command="tail -f /dev/null",
291+
command=command,
316292
nano_cpus=nano_cpus,
317293
detach=True,
318294
)

commit0/harness/evaluate.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ def main(
2626
branch: str,
2727
backend: str,
2828
timeout: int,
29+
num_cpus: int,
2930
num_workers: int,
3031
) -> None:
3132
dataset: Iterator[RepoInstance] = load_dataset(dataset_name, split=dataset_split) # type: ignore
@@ -52,6 +53,7 @@ def main(
5253
test_dir,
5354
backend,
5455
timeout,
56+
num_cpus,
5557
stdout=False,
5658
): None
5759
for repo, test_dir in pairs

commit0/harness/execution_context.py

Lines changed: 74 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import modal.io_streams
1212
from enum import StrEnum, auto
1313
from pathlib import Path
14+
import time
1415
from typing import Optional, Type
1516
from types import TracebackType
1617

@@ -27,7 +28,6 @@
2728
create_container,
2829
copy_from_container,
2930
copy_to_container,
30-
delete_file_from_container,
3131
exec_run_with_timeout,
3232
)
3333

@@ -51,43 +51,29 @@ def __init__(
5151
spec: Spec,
5252
logger: logging.Logger,
5353
timeout: int,
54+
num_cpus: int,
55+
log_dir: Path,
56+
files_to_copy: Optional[Files] = None,
5457
):
5558
"""Create the remote execution context
5659
57-
The execution context will persist for the lifetime of this object.
5860
The execution context can be a Docker container or Modal sandbox.
61+
The execution context may not persist for the lifetime of this object.
5962
"""
6063
self.spec = spec
6164
self.logger = logger
6265
self.timeout = timeout
66+
self.num_cpus = num_cpus
67+
self.log_dir = log_dir
6368

6469
@abstractmethod
65-
def exec_run_with_timeout(
66-
self, command: str, timeout: int
67-
) -> tuple[str, bool, float]:
68-
"""Exec"""
69-
raise NotImplementedError
70-
71-
@abstractmethod
72-
def exec_run(self, command: str) -> tuple[int, str]:
73-
"""Exec"""
74-
raise NotImplementedError
75-
76-
@abstractmethod
77-
def copy_from_remote(self, remote_path: Path, local_path: Path) -> None:
78-
"""Copy"""
79-
raise NotImplementedError
80-
81-
@abstractmethod
82-
def delete_file_from_remote(self, remote_path: Path) -> None:
83-
"""Delete"""
70+
def exec_run_with_timeout(self, command: str) -> tuple[str, bool, float]:
71+
"""Execute a test command"""
8472
raise NotImplementedError
8573

86-
def write_test_output(
87-
self, log_dir: Path, test_output: str, timed_out: bool
88-
) -> None:
74+
def write_test_output(self, test_output: str, timed_out: bool) -> None:
8975
"""Write test output"""
90-
test_output_path = log_dir / "test_output.txt"
76+
test_output_path = self.log_dir / "test_output.txt"
9177
with open(test_output_path, "w") as f:
9278
f.write(test_output)
9379
if timed_out:
@@ -98,15 +84,6 @@ def write_test_output(
9884
self.logger,
9985
)
10086

101-
# copy back report.json if there is any
102-
report_file = Path(self.spec.repo_directory) / "report.json"
103-
# Run the test command inside the container to check if the file exists
104-
exit_code, output = self.exec_run(f"test -e {report_file}")
105-
# Check the exit code of the command
106-
if exit_code == 0:
107-
self.copy_from_remote(report_file, log_dir / "report.json")
108-
self.delete_file_from_remote(report_file)
109-
11087
def __enter__(self):
11188
return self
11289

@@ -126,39 +103,41 @@ def __init__(
126103
spec: Spec,
127104
logger: logging.Logger,
128105
timeout: int,
106+
num_cpus: int,
107+
log_dir: Path,
129108
files_to_copy: Optional[Files] = None,
130109
):
131-
super().__init__(spec, logger, timeout)
110+
super().__init__(spec, logger, timeout, num_cpus, log_dir)
132111

133112
self.client = docker.from_env()
134113
self.container = create_container(
135114
client=self.client,
136115
image_name=spec.repo_image_key,
137116
container_name=spec.get_container_name(),
117+
nano_cpus=num_cpus,
138118
logger=logger,
139119
)
140120
self.container.start()
141121
if files_to_copy:
142122
for _, f in files_to_copy.items():
143123
copy_to_container(self.container, f["src"], f["dest"]) # type: ignore
144124

145-
def exec_run_with_timeout(
146-
self, command: str, timeout: int
147-
) -> tuple[str, bool, float]:
148-
"""Exec"""
149-
return exec_run_with_timeout(self.container, command, timeout)
150-
151-
def exec_run(self, command: str) -> tuple[int, str]:
125+
def exec_run_with_timeout(self, command: str) -> tuple[str, bool, float]:
152126
"""Exec"""
153-
return self.container.exec_run(command, demux=True)
154-
155-
def copy_from_remote(self, remote_path: Path, local_path: Path) -> None:
156-
"""Copy"""
157-
copy_from_container(self.container, remote_path, local_path)
127+
output = exec_run_with_timeout(self.container, command, self.timeout)
158128

159-
def delete_file_from_remote(self, remote_path: Path) -> None:
160-
"""Delete"""
161-
delete_file_from_container(self.container, str(remote_path))
129+
# copy back report.json if there is any
130+
report_file = Path(self.spec.repo_directory) / "report.json"
131+
# Run the test command inside the container to check if the file exists
132+
exit_code, test_output = self.container.exec_run(
133+
f"test -e {report_file}", demux=True
134+
)
135+
# Check the exit code of the command
136+
if exit_code == 0:
137+
copy_from_container(
138+
self.container, report_file, self.log_dir / "report.json"
139+
)
140+
return output
162141

163142
def __exit__(
164143
self,
@@ -176,64 +155,67 @@ def __init__(
176155
spec: Spec,
177156
logger: logging.Logger,
178157
timeout: int,
158+
num_cpus: int,
159+
log_dir: Path,
179160
files_to_copy: Optional[Files] = None,
180161
):
181-
super().__init__(spec, logger, timeout)
162+
super().__init__(spec, logger, timeout, num_cpus, log_dir)
163+
164+
self.app = modal.App()
182165

183166
# the image must exist on dockerhub
184167
reponame = spec.repo.split("/")[-1]
185-
image_name = f"wentingzhao/{reponame}"
168+
image_name = f"wentingzhao/{reponame}:latest"
186169
image = modal.Image.from_registry(image_name)
187170
if files_to_copy:
188171
for _, f in files_to_copy.items():
189172
image = image.copy_local_file(f["src"], f["dest"]) # type: ignore
173+
self.image = image
190174

191-
self.sandbox = modal.Sandbox.create(
192-
"sleep",
193-
"infinity",
194-
image=image,
195-
cpu=4.0,
196-
timeout=timeout,
197-
)
198-
199-
def exec_run_with_timeout(
200-
self, command: str, timeout: int
201-
) -> tuple[str, bool, float]:
202-
"""Execute command on modal sandbox"""
203-
print("Executing:", command)
204-
process = self.sandbox.exec("bash", "-c", command)
205-
print("stdout")
206-
stdout = read_stream(process.stdout)
207-
print("stderr")
208-
stderr = read_stream(process.stderr)
209-
print(stderr)
210-
return stdout, False, 1.0
211-
return stdout, stderr
212-
213-
def exec_run(self, command: str) -> tuple[int, str]:
175+
def exec_run_with_timeout(self, command: str) -> tuple[str, bool, float]:
214176
"""Execute command on modal sandbox"""
215-
process = self.sandbox.exec("bash", "-c", command)
216-
stdout = read_stream(process.stdout)
217-
stderr = read_stream(process.stderr)
218-
print(stderr)
219-
return 1, stdout
220-
221-
def copy_from_remote(self, remote_path: Path, local_path: Path) -> None:
222-
"""Copy file from modal sandbox"""
223-
process = self.sandbox.exec("bash", "-c", f"cat {str(remote_path)}")
224-
output = "".join([line for line in process.stdout]).strip()
225-
with local_path.open("w") as f:
226-
f.write(output)
227-
228-
def delete_file_from_remote(self, remote_path: Path) -> None:
229-
"""Delete"""
230-
self.sandbox.exec("bash", "-c", f"rm {str(remote_path)}")
177+
start_time = time.time()
178+
with modal.Volume.ephemeral() as vol:
179+
# copy back report.json if there is any
180+
report_file = Path(self.spec.repo_directory) / "report.json"
181+
182+
self.sandbox = modal.Sandbox.create(
183+
"bash",
184+
"-c",
185+
f"{command} && cp {str(report_file)} /vol/report.json",
186+
image=self.image,
187+
cpu=self.num_cpus,
188+
timeout=self.timeout,
189+
app=self.app,
190+
volumes={"/vol": vol},
191+
)
192+
self.sandbox.wait()
193+
194+
# stdout has been redirected to stderr
195+
stdout = read_stream(self.sandbox.stderr)
196+
197+
return_code = self.sandbox.returncode
198+
# https://github.com/modal-labs/modal-client/blob/d577b2916b5c3bf4ebbcb58fadced84d85e1cf8c/modal/sandbox.py#L413
199+
if return_code == 124:
200+
timed_out = True
201+
else:
202+
timed_out = False
203+
204+
# copy over report.json from mount
205+
with (self.log_dir / "report.json").open("wb") as f:
206+
for data in vol.read_file("report.json"):
207+
f.write(data)
208+
209+
self.sandbox.terminate()
210+
211+
end_time = time.time()
212+
213+
return stdout, timed_out, end_time - start_time
231214

232215
def __exit__(
233216
self,
234217
exctype: Optional[Type[BaseException]],
235218
excinst: Optional[BaseException],
236219
exctb: Optional[TracebackType],
237220
) -> None:
238-
self.sandbox.terminate()
239221
close_logger(self.logger)

commit0/harness/run_pytest_ids.py

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ def main(
3636
test_ids: str,
3737
backend: str,
3838
timeout: int,
39+
num_cpus: int,
3940
stdout: bool,
4041
) -> str:
4142
"""Runs the pytests for repos in a dataset.
@@ -92,16 +93,19 @@ def main(
9293
)
9394

9495
try:
95-
with execution_context(spec, logger, timeout, files_to_copy) as context:
96+
with execution_context(
97+
spec, logger, timeout, num_cpus, log_dir, files_to_copy
98+
) as context:
9699
output, timed_out, total_runtime = context.exec_run_with_timeout(
97-
"/bin/bash /eval.sh", timeout
100+
"/bin/bash /eval.sh"
98101
)
99102
logger.info(output)
100103
test_output = extract_test_output(
101104
output, "--json-report --json-report-file=report.json"
102105
)
103-
context.write_test_output(log_dir, test_output, timed_out)
104-
print(test_output)
106+
context.write_test_output(test_output, timed_out)
107+
if stdout:
108+
print(test_output)
105109
except EvaluationError as e:
106110
error_msg = (
107111
f"Error in running pytest for {repo}: {e}\n"

commit0/harness/spec.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,8 @@ def make_eval_script_list(instance: RepoInstance, repo_directory: str) -> list[s
157157
f"git reset --hard {instance['base_commit']}",
158158
"git status",
159159
]
160+
for i in range(len(eval_script_list)):
161+
eval_script_list[i] = f"{eval_script_list[i]} 1>&2"
160162
return eval_script_list
161163

162164

0 commit comments

Comments
 (0)