Merge branch 'main' into aider

Author: wenting-zhao

.github/workflows/pre-commit.yml

@@ -10,5 +10,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-python@v3
-      - uses: pre-commit/[email protected]
+      - name: Install uv
+        uses: astral-sh/setup-uv@v2
+      - name: Set up Python
+        run: uv venv --python 3.12
+      - name: Install the project
+        run: uv sync
+      - name: Install the project
+        run: uv pip install pre-commit
+      - name: PreCommit
+        run: uv run pre-commit run --show-diff-on-failure --color=always --all-files

.github/workflows/system.yml

@@ -0,0 +1,26 @@
+name: system
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  system:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install uv
+        uses: astral-sh/setup-uv@v2
+      - name: Set up Python
+        run: uv venv --python 3.12
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v3
+      - name: Install the project
+        run: uv sync
+      - name: Clone
+        run: uv run commit0 clone simpy
+      - name: Setup
+        run: uv run commit0 build simpy
+      - name: Test
+        run: uv run commit0 test-reference simpy tests/test_event.py::test_succeed

commit0/__main__.py

@@ -1,27 +1,70 @@
-import argparse
 import commit0.harness.run_pytest_ids
 import commit0.harness.build
 import commit0.harness.setup
+import copy
+import sys
+import os
+import hydra
+from hydra.core.config_store import ConfigStore
+from commit0.configs.config_class import Commit0Config
+from commit0.harness.constants import COMMANDS, SPLIT
 
 
 def main() -> None:
-    parser = argparse.ArgumentParser(description="Commit0 version control system")
-    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+    command = sys.argv[1]
+    if command not in COMMANDS:
+        raise ValueError(
+            f"command must be from {', '.join(COMMANDS)}, but you provided {command}"
+        )
+    # type check config values
+    cs = ConfigStore.instance()
+    cs.store(name="user", node=Commit0Config)
+    # have hydra to ignore all command-line arguments
+    sys_argv = copy.deepcopy(sys.argv)
+    sys.argv = [sys.argv[0]]
+    hydra.initialize(version_base=None, config_path="configs")
+    config = hydra.compose(config_name="user")
+    # after hydra gets all configs, put command-line arguments back
+    sys.argv = sys_argv
+    # repo_split: split from command line has a higher priority than split in hydra
+    if command in ["clone", "build"]:
+        if len(sys.argv) == 3:
+            if sys.argv[2] not in SPLIT:
+                raise ValueError(
+                    f"repo split must be from {', '.join(SPLIT.keys())}, but you provided {sys.argv[2]}"
+                )
+            config.repo_split = sys.argv[2]
+    config.base_dir = os.path.abspath(config.base_dir)
 
-    commit0.harness.setup.add_init_args(subparsers.add_parser("clone"))
-    commit0.harness.build.add_init_args(subparsers.add_parser("build"))
-    commit0.harness.run_pytest_ids.add_init_args(subparsers.add_parser("test"))
-
-    args = parser.parse_args()
-
-    if args.command == "clone":
-        commit0.harness.setup.run(args)
-    elif args.command == "build":
-        commit0.harness.build.run(args)
-    elif args.command == "test":
-        commit0.harness.run_pytest_ids.run(args)
-    else:
-        parser.print_help()
+    if command == "clone":
+        commit0.harness.setup.main(
+            config.dataset_name,
+            config.dataset_split,
+            config.repo_split,
+            config.base_dir,
+        )
+    elif command == "build":
+        commit0.harness.build.main(
+            config.dataset_name,
+            config.dataset_split,
+            config.repo_split,
+            config.num_workers,
+        )
+    elif command == "test" or command == "test-reference":
+        repo = sys.argv[2]
+        test_ids = sys.argv[3]
+        if command == "test-reference":
+            config.branch = "reference"
+        commit0.harness.run_pytest_ids.main(
+            config.dataset_name,
+            config.dataset_split,
+            config.base_dir,
+            repo,
+            config.branch,
+            test_ids,
+            config.backend,
+            config.timeout,
+        )
 
 
 if __name__ == "__main__":

commit0/configs/base.yaml

@@ -0,0 +1,18 @@
+defaults:
+  - _self_
+
+# shared in all steps
+dataset_name: wentingzhao/commit0_docstring
+dataset_split: test
+
+# clone related
+base_dir: repos/
+repo_split: all
+
+# build related
+num_workers: 8
+
+# test related
+backend: local
+branch: ai
+timeout: 1_800

commit0/configs/config_class.py

@@ -0,0 +1,23 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class Commit0Config:
+    # shared in all steps
+    dataset_name: str
+    dataset_split: str
+
+    # clone related
+    base_dir: str
+    repo_split: str
+
+    # build related
+    # which repo to build, all or one repo
+    num_workers: int
+
+    # test related
+    backend: str
+    # which branch to work on
+    branch: str
+    # timeout for running pytest
+    timeout: int

commit0/configs/user.yaml

@@ -0,0 +1,3 @@
+defaults:
+  - base
+  - _self_

commit0/harness/build.py

@@ -1,11 +1,12 @@
-import argparse
 import logging
 
 import docker
 from datasets import load_dataset
 from typing import Iterator
+
 from commit0.harness.docker_build import build_repo_images
 from commit0.harness.spec import make_spec
+from commit0.harness.constants import RepoInstance, SPLIT
 
 logging.basicConfig(
     level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
@@ -14,49 +15,19 @@
 
 
 def main(
-    hf_name: str,
-    base_dir: str,
-    config_file: str,
+    dataset_name: str, dataset_split: str, repo_split: str, num_workers: int
 ) -> None:
-    dataset: Iterator[RepoInstance] = load_dataset(hf_name, split="test")
+    dataset: Iterator[RepoInstance] = load_dataset(dataset_name, split=dataset_split)  # type: ignore
     specs = []
     for example in dataset:
+        repo_name = example["repo"].split("/")[-1]
+        if repo_split != "all" and repo_name not in SPLIT[repo_split]:
+            continue
         spec = make_spec(example)
         specs.append(spec)
 
     client = docker.from_env()
-    build_repo_images(client, specs)
-    logger.info("Done building docker images")
-
-
-def add_init_args(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument(
-        "--hf_name",
-        type=str,
-        help="HF dataset name",
-        default="wentingzhao/commit0_docstring",
-    )
-    parser.add_argument(
-        "--base_dir",
-        type=str,
-        default="repos/",
-        help="base directory to write repos to",
-    )
-    parser.add_argument(
-        "--config_file",
-        type=str,
-        default="config.yml",
-        help="where to write config file to",
-    )
-    parser.set_defaults(func=run)
-
-
-def run(args: argparse.Namespace) -> None:
-    main(
-        hf_name=args.hf_name,
-        base_dir=args.base_dir,
-        config_file=args.config_file,
-    )
+    build_repo_images(client, specs, num_workers)
 
 
 __all__ = []

commit0/harness/constants.py

@@ -1,14 +1,14 @@
 from enum import Enum
 from pathlib import Path
-from typing import TypedDict
+from typing import Dict, TypedDict
 
 
 class RepoInstance(TypedDict):
     repo: str
     base_commit: str
     reference_commit: str
     setup: dict
-    test: str
+    test: Dict[str, str]
 
 
 # Constants - Evaluation Log Directories
@@ -25,6 +25,88 @@ class RepoInstance(TypedDict):
 # Evaluation backends
 EVAL_BACKENDS = ["local", "modal"]
 
+# available commands
+COMMANDS = ["clone", "build", "test", "test-reference"]
+# repo splits
+SPLIT_MINITORCH = ["minitorch"]
+SPLIT_SIMPY = ["simpy"]
+SPLIT_LITE = [
+    "tinydb",
+    "simpy",
+    "deprecated",
+    "wcwidth",
+    "voluptuous",
+    "cachetools",
+    "imapclient",
+    "marshmallow",
+    "jinja",
+    "cookiecutter",
+]
+SPLIT_ALL = [
+    "statsmodels",
+    "python-progressbar",
+    "xarray",
+    "imbalanced-learn",
+    "web3.py",
+    "scrapy",
+    "seaborn",
+    "pypdf",
+    "pexpect",
+    "pytest",
+    "pylint",
+    "joblib",
+    "dulwich",
+    "virtualenv",
+    "minitorch",
+    "networkx",
+    "requests",
+    "sphinx",
+    "jedi",
+    "moviepy",
+    "loguru",
+    "paramiko",
+    "geopandas",
+    "bitstring",
+    "fastapi",
+    "chardet",
+    "tornado",
+    "python-prompt-toolkit",
+    "attrs",
+    "PyBoy",
+    "pydantic",
+    "filesystem_spec",
+    "tlslite-ng",
+    "graphene",
+    "mimesis",
+    "babel",
+    "dnspython",
+    "portalocker," "cookiecutter",
+    "pyjwt",
+    "python-rsa",
+    "more-itertools",
+    "simpy",
+    "click",
+    "fabric",
+    "jinja",
+    "flask",
+    "sqlparse",
+    "marshmallow",
+    "imapclient",
+    "tinydb",
+    "cachetools",
+    "voluptuous",
+    "parsel",
+    "wcwidth",
+    "deprecated",
+]
+
+SPLIT = {
+    "all": SPLIT_ALL,
+    "minitorch": SPLIT_MINITORCH,
+    "simpy": SPLIT_SIMPY,
+    "lite": SPLIT_LITE,
+}
+
 
 class ResolvedStatus(Enum):
     NO = "RESOLVED_NO"