Merge branch 'develop' into tawsif/fix-imports

Author: tawsifkamal

.codegen/.gitignore

@@ -8,12 +8,8 @@ jupyter/
 codegen-system-prompt.txt
 
 # Python cache files
-__pycache__/
+**/__pycache__/
 *.py[cod]
 *$py.class
 *.txt
 *.pyc
-
-# Keep codemods
-!codemods/
-!codemods/**

.codegen/codemods/no_link_backticks/no_link_backticks.py

@@ -2,7 +2,7 @@
 from codegen import Codebase
 
 
-@codegen.function("no-link-backticks")
+@codegen.function(name="no-link-backticks", subdirectories=["test/unit"])
 def run(codebase: Codebase):
     import re
 
@@ -12,6 +12,7 @@ def run(codebase: Codebase):
     # Iterate over all .mdx files in the codebase
     for file in codebase.files(extensions=["mdx"]):
         if file.extension == ".mdx":
+            print(f"Processing {file.path}")
             new_content = file.content
 
             # Find all markdown links with backticks in link text

.github/workflows/generate-docs.yml

@@ -24,11 +24,14 @@ jobs:
       - name: Generate API reference
         run: uv run python src/codegen/gscli/cli.py generate docs
 
+      - name: Generate System Prompt
+        run: uv run python src/codegen/gscli/cli.py generate system-prompt
+
       - name: Commit changes
         run: |
           git config --local user.email ${{ secrets.DOCS_USER_EMAIL }}
           git config --local user.name ${{ secrets.DOCS_USER_NAME }}
-          git add docs/
+          git add docs/ src/codegen/sdk/system-prompt.txt
           git diff --staged --quiet || git commit -m "docs: updated API reference"
 
       - name: Push changes

.github/workflows/test.yml

@@ -23,7 +23,6 @@ jobs:
   unit-tests:
     needs: access-check
     runs-on: ubuntu-latest-8
-    environment: testing
     steps:
       - uses: actions/checkout@v4
         with:
@@ -48,7 +47,6 @@ jobs:
     # TODO: re-enable when this check is a develop required check
     if: false
     runs-on: ubuntu-latest-32
-    environment: testing
     strategy:
       matrix:
         sync_graph: [ true, false ]
@@ -90,7 +88,6 @@ jobs:
     needs: access-check
     if: contains(github.event.pull_request.labels.*.name, 'parse-tests') || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest-32
-    environment: testing
     steps:
       - uses: actions/checkout@v4
         with:
@@ -161,7 +158,6 @@ jobs:
   integration-tests:
     needs: access-check
     runs-on: ubuntu-latest-16
-    environment: testing
     steps:
       - uses: actions/checkout@v4
         with:

codegen-examples/examples/swebench_agent_run/.env.template

@@ -2,3 +2,5 @@ OPENAI_API_KEY= # Your OpenAI API key
 ANTHROPIC_API_KEY= # Your Anthropic API key
 LANGSMITH_API_KEY= # Your Langsmith API key
 LANGCHAIN_TRACING_V2= # `true` for tracing, `false` for no tracing
+LANGCHAIN_PROJECT= # Your Langchain project
+RELACE_API= # Your Relace API key

codegen-examples/examples/swebench_agent_run/entry_point.py

@@ -1,10 +1,12 @@
 from codegen.extensions.swebench.utils import SweBenchExample
 from codegen.extensions.swebench.harness import run_agent_on_entry
 import modal
+import sys
+from codegen.sdk.core.codebase import Codebase
 
 image = (
     modal.Image.debian_slim(python_version="3.13")
-    .apt_install("git")
+    .apt_install(["git", "ripgrep"])
     .pip_install("fastapi[standard]")
     .copy_local_dir("../../../", "/root/codegen", ignore=[".venv", "**/.venv", "tests", "**/tests"])
     .run_commands("pip install -e /root/codegen")
@@ -17,3 +19,22 @@
 async def run_agent_modal(entry: SweBenchExample):
     """Modal function to process a single example from the SWE-bench dataset."""
     return run_agent_on_entry(entry)
+
+
+@app.cls(image=image, secrets=[modal.Secret.from_dotenv()], enable_memory_snapshot=True)
+class SwebenchAgentRun:
+    repo_full_name: str = modal.parameter()
+    commit: str = modal.parameter()
+    codebase: Codebase | None = None
+
+    @modal.enter(snap=True)
+    def load(self):
+        self.codebase = Codebase.from_repo(repo_full_name=self.repo_full_name, commit=self.commit, language="python")
+
+    @modal.exit()
+    def exit(self):
+        sys.exit(0)
+
+    @modal.method()
+    async def run(self, entry: SweBenchExample):
+        return run_agent_on_entry(entry, codebase=self.codebase)

codegen-examples/examples/swebench_agent_run/run_eval.py

@@ -6,16 +6,16 @@
 import modal
 import click
 from datetime import datetime
-from codegen.extensions.swebench.utils import SWEBenchDataset, get_swe_bench_example, get_swe_bench_examples
+from codegen.extensions.swebench.utils import SWEBenchDataset, SweBenchExample, get_swe_bench_example, get_swe_bench_examples
 from codegen.extensions.swebench.report import generate_report
 
 PREDS_DNAME = Path(__file__).parent / "predictions"
 LOG_DIR = Path(__file__).parent / "logs"
 
-run_agent_modal = modal.Function.lookup("swebench-agent-run", "run_agent_modal")
+SwebenchAgentRun = modal.Cls.from_name(app_name="swebench-agent-run", name="SwebenchAgentRun")
 
 
-async def process_batch(examples, batch_size=10):
+async def process_batch(examples: list[SweBenchExample], batch_size=10):
     """Process a batch of examples concurrently.
 
     Args:
@@ -31,7 +31,7 @@ async def process_batch(examples, batch_size=10):
         batch = examples[i : i + batch_size]
 
         # Create tasks for this batch
-        batch_tasks = [run_agent_modal.remote.aio(example) for example in batch]
+        batch_tasks = [SwebenchAgentRun(repo_full_name=example.repo, commit=example.base_commit).run.remote.aio(example) for example in batch]
 
         # Wait for all tasks in this batch to complete
         print(f"Processing batch {i // batch_size + 1}/{len(examples) // batch_size + 1} (examples {i + 1}-{min(i + batch_size, len(examples))})")

codegen-examples/examples/swebench_agent_run/test.py

@@ -0,0 +1,14 @@
+from codegen import Codebase
+import modal
+
+image = modal.Image.debian_slim(python_version="3.13").apt_install("git").pip_install("fastapi[standard]").run_commands("pip install codegen")
+
+app = modal.App(name="codegen-examples", image=image, secrets=[modal.Secret.from_dotenv()])
+
+
+@app.function()
+def run_agent(AgentClass):
+    codebase = Codebase.from_repo(repo_full_name="pallets/flask")
+    agent = AgentClass(codebase)
+    agent.run(prompt="Tell me about the codebase and the files in it.")
+    return True

docs/changelog/changelog.mdx

@@ -4,6 +4,74 @@ icon: "clock"
 iconType: "solid"
 ---
 
+<Update label="v0.40.0" description="February 26, 2025">
+### [Adds Chat Agent and subdirectory support in codegen.](https://github.com/codegen-sh/codegen-sdk/releases/tag/v0.40.0)
+- Add functionality to specify subdirectories in `codegen.function`
+- Introduce a new Chat Agent
+</Update>
+
+<Update label="v0.39.1" description="February 26, 2025">
+### [Custom codebase path setting for code generation.](https://github.com/codegen-sh/codegen-sdk/releases/tag/v0.39.1)
+- Allow setting custom codebase path in code generation
+</Update>
+
+<Update label="v0.39.0" description="February 26, 2025">
+### [Lazy Graph Compute Mode and new agent run snapshots.](https://github.com/codegen-sh/codegen-sdk/releases/tag/v0.39.0)
+- Implement Lazy Graph Compute Mode
+- Add agent run snapshots feature
+- Update API reference documentation
+</Update>
+
+<Update label="v0.38.0" description="February 26, 2025">
+### [Adds ripgrep search tool and updates API docs.](https://github.com/codegen-sh/codegen-sdk/releases/tag/v0.38.0)
+- Integrate ripgrep in search tool
+- Update API reference documentation
+</Update>
+
+
+<Update label="v0.37.1" description="February 26, 2025">
+### [Fixes linting issues.](https://github.com/codegen-sh/codegen-sdk/releases/tag/v0.37.1)
+- Fix linting issues
+</Update>
+
+
+<Update label="v0.37.0" description="February 26, 2025">
+### [Custom package resolving and error handling improvements.](https://github.com/codegen-sh/codegen-sdk/releases/tag/v0.37.0)
+- Allow custom overrides for package resolving
+- Add optional sys.path support
+- Improve error handling with linear tools
+- Fix wildcard resolution issues
+</Update>
+
+
+<Update label="v0.36.0" description="February 25, 2025">
+### [File Parse Mode disabled by default](https://github.com/codegen-sh/codegen-sdk/releases/tag/v0.36.0)
+- Disable File Parse Mode feature implemented
+</Update>
+
+
+<Update label="v0.35.0" description="February 25, 2025">
+### [Replaces edit tool and enhances test configurations.](https://github.com/codegen-sh/codegen-sdk/releases/tag/v0.35.0)
+- Replace edit tool with a new version
+- Enhance test configurations and execution
+</Update>
+
+
+<Update label="v0.34.0" description="February 24, 2025">
+### [Introduces Docker client and major refactor.](https://github.com/codegen-sh/codegen-sdk/releases/tag/v0.34.0)
+- Introduce Docker client feature
+- Refactor codebase directory structure
+- Implement heuristics-based minified file detection
+</Update>
+
+
+<Update label="v0.33.1" description="February 24, 2025">
+### [Improves GitHub Actions handling and fixes Docker issues.](https://github.com/codegen-sh/codegen-sdk/releases/tag/v0.33.1)
+- Improve GitHub Actions token and repo handling
+- Fix codegen Docker runner issues
+- Update agent documentation
+</Update>
+
 <Update label="v0.33.0" description="February 24, 2025">
 ### [Introduces CodegenApp and enhances codegen capabilities.](https://github.com/codegen-sh/codegen-sdk/releases/tag/v0.33.0)
 - Introduce CodegenApp feature

src/codegen/agents/chat_agent.py

@@ -0,0 +1,95 @@
+from typing import TYPE_CHECKING, Optional
+from uuid import uuid4
+
+from langchain.tools import BaseTool
+from langchain_core.messages import AIMessage
+
+from codegen.extensions.langchain.agent import create_chat_agent
+
+if TYPE_CHECKING:
+    from codegen import Codebase
+
+
+class ChatAgent:
+    """Agent for interacting with a codebase."""
+
+    def __init__(self, codebase: "Codebase", model_provider: str = "anthropic", model_name: str = "claude-3-5-sonnet-latest", memory: bool = True, tools: Optional[list[BaseTool]] = None, **kwargs):
+        """Initialize a CodeAgent.
+
+        Args:
+            codebase: The codebase to operate on
+            model_provider: The model provider to use ("anthropic" or "openai")
+            model_name: Name of the model to use
+            memory: Whether to let LLM keep track of the conversation history
+            tools: Additional tools to use
+            **kwargs: Additional LLM configuration options. Supported options:
+                - temperature: Temperature parameter (0-1)
+                - top_p: Top-p sampling parameter (0-1)
+                - top_k: Top-k sampling parameter (>= 1)
+                - max_tokens: Maximum number of tokens to generate
+        """
+        self.codebase = codebase
+        self.agent = create_chat_agent(self.codebase, model_provider=model_provider, model_name=model_name, memory=memory, additional_tools=tools, **kwargs)
+
+    def run(self, prompt: str, thread_id: Optional[str] = None) -> str:
+        """Run the agent with a prompt.
+
+        Args:
+            prompt: The prompt to run
+            thread_id: Optional thread ID for message history. If None, a new thread is created.
+
+        Returns:
+            The agent's response
+        """
+        if thread_id is None:
+            thread_id = str(uuid4())
+
+        input = {"messages": [("user", prompt)]}
+        stream = self.agent.stream(input, config={"configurable": {"thread_id": thread_id}}, stream_mode="values")
+
+        for s in stream:
+            message = s["messages"][-1]
+            if isinstance(message, tuple):
+                print(message)
+            else:
+                if isinstance(message, AIMessage) and isinstance(message.content, list) and "text" in message.content[0]:
+                    AIMessage(message.content[0]["text"]).pretty_print()
+                else:
+                    message.pretty_print()
+
+        return s["messages"][-1].content
+
+    def chat(self, prompt: str, thread_id: Optional[str] = None) -> tuple[str, str]:
+        """Chat with the agent, maintaining conversation history.
+
+        Args:
+            prompt: The user message
+            thread_id: Optional thread ID for message history. If None, a new thread is created.
+
+        Returns:
+            A tuple of (response_content, thread_id) to allow continued conversation
+        """
+        if thread_id is None:
+            thread_id = str(uuid4())
+            print(f"Starting new chat thread: {thread_id}")
+        else:
+            print(f"Continuing chat thread: {thread_id}")
+
+        response = self.run(prompt, thread_id=thread_id)
+        return response, thread_id
+
+    def get_chat_history(self, thread_id: str) -> list:
+        """Retrieve the chat history for a specific thread.
+
+        Args:
+            thread_id: The thread ID to retrieve history for
+
+        Returns:
+            List of messages in the conversation history
+        """
+        # Access the agent's memory to get conversation history
+        if hasattr(self.agent, "get_state"):
+            state = self.agent.get_state({"configurable": {"thread_id": thread_id}})
+            if state and "messages" in state:
+                return state["messages"]
+        return []

src/codegen/cli/commands/list/main.py

@@ -17,10 +17,16 @@ def list_command():
         table.add_column("Name", style="cyan")
         table.add_column("Type", style="magenta")
         table.add_column("Path", style="dim")
+        table.add_column("Subdirectories", style="dim")
 
         for func in functions:
             func_type = "Webhook" if func.lint_mode else "Function"
-            table.add_row(func.name, func_type, str(func.filepath.relative_to(Path.cwd())) if func.filepath else "<unknown>")
+            table.add_row(
+                func.name,
+                func_type,
+                str(func.filepath.relative_to(Path.cwd())) if func.filepath else "<unknown>",
+                ", ".join(func.subdirectories) if func.subdirectories else "",
+            )
 
         rich.print(table)
         rich.print("\nRun a function with:")