fixes formatting

Author: wenting-zhao

commit0/cli.py

@@ -2,6 +2,7 @@
 from pathlib import Path
 from typing import Union, List
 from typing_extensions import Annotated
+import commit0.harness.batch_run_pytest_ids
 import commit0.harness.run_pytest_ids
 import commit0.harness.get_pytest_ids
 import commit0.harness.build
@@ -300,6 +301,49 @@ def test(
     )
 
 
+@commit0_app.command()
+def batch_test(
+    test_ids: str = typer.Argument(
+        None,
+        help='All ways pytest supports to run and select tests. Please provide a single string. Example: "test_mod.py", "testing/", "test_mod.py::test_func", "-k \'MyClass and not method\'"',
+    ),
+    backend: str = typer.Option("modal", help="Backend to use for testing"),
+    timeout: int = typer.Option(1800, help="Timeout for tests in seconds"),
+    num_cpus: int = typer.Option(1, help="Number of CPUs to use"),
+    reference: Annotated[
+        bool, typer.Option("--reference", help="Test the reference commit")
+    ] = False,
+    coverage: Annotated[
+        bool, typer.Option("--coverage", help="Whether to get coverage information")
+    ] = False,
+    rebuild: bool = typer.Option(
+        False, "--rebuild", help="Whether to rebuild an image"
+    ),
+    commit0_config_file: str = typer.Option(
+        ".commit0.yaml",
+        help="Path to the commit0 dot file, where the setup config is stored",
+    ),
+    verbose: int = typer.Option(
+        1,
+        "--verbose",
+        "-v",
+        help="Set this to 2 for more logging information",
+        count=True,
+    ),
+) -> None:
+    """Run tests on a Commit0 repository."""
+    commit0.harness.batch_run_pytest_ids.main(
+        test_ids,
+        reference,
+        coverage,
+        backend,
+        timeout,
+        num_cpus,
+        rebuild,
+        verbose,
+    )
+
+
 @commit0_app.command()
 def evaluate(
     branch: Union[str, None] = typer.Option(

commit0/harness/utils.py

@@ -239,7 +239,7 @@ def extract_code_blocks(text: str) -> List[str]:
                    from the text.
 
     """
-    pattern = r"```python\n(.*?)```"
+    pattern = r'(?s)```(?:python|py)?(.*?)```'
     matches = re.finditer(pattern, text, re.DOTALL)
     return [match.group(1).strip() for match in matches]
 

examples/star/inference.py

@@ -1,4 +1,4 @@
-from typing import List
+from typing import Dict, List
 from datasets import Dataset
 from vllm import LLM, SamplingParams
 from examples.star.utils import generate_prompt, cleanup
@@ -28,14 +28,14 @@ def generate_predictions(
     sampling_params = SamplingParams(n=n, temperature=temperature, max_tokens=512)
     llm = LLM(model=model_name)
 
-    prompts: List[str] = []
+    prompts: List[List[Dict]] = []
     for example in dataset:
         prompt = example["prompt"]
         test = example["test"]
         prompt = generate_prompt(prompt, test)
-        prompts.append(prompt)
+        prompts.append([{"role": "user", "content": prompt}])
 
-    outputs = llm.generate(prompts, sampling_params)
+    outputs = llm.chat(prompts, sampling_params)
 
     results: List[List[str]] = []
     for output in outputs:

examples/star/run.sh

@@ -1,14 +1,15 @@
 python examples/star/star.py \
   --model_name_or_path meta-llama/Llama-3.1-8B-Instruct \
   --dataset_name commit0/mbpp \
-  -n 10 \
+  -n 16 \
   --output_dir outputs \
   --low_cpu_mem_usage \
   --with_tracking \
   --report_to wandb \
   --iteration 5 \
   --learning_rate 1e-6 \
   --per_device_train_batch_size 1 \
-  --gradient_accumulation_steps 8 \
-  --max_workers 64
+  --gradient_accumulation_steps 128 \
+  --max_workers 64 \
+  --temperature 1
 

pyproject.toml

@@ -36,14 +36,15 @@ packages = ["commit0", "agent"]
 agent = [
     "import-deps>=0.3.0",
     "PyMuPDF>=1.24.5",
-    "aider-chat",
+    #"aider-chat>=0.69.1",
 ]
 example = [
-    "vllm",
-    "transformers",
+    "vllm==0.6.5",
+    "transformers>=4.47.1",
     "setuptools",
     "accelerate",
     "wandb>=0.19.0",
+    "openai>=1.58.1",
 ]
 
 [project.urls]