Skip to content

Add allow_external flag & Fix Tests #807

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Mar 12, 2025
13 changes: 13 additions & 0 deletions docs/introduction/advanced-settings.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,19 @@ Controls import path overrides during import resolution.

Enables and disables resolution of imports from `sys.path`.

<Note>
For this to properly work, you may also need to enable `allow_external`.
</Note>

## Flag: `allow_external`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hm I don't think we have the performance in the current sdk to handle external resolution so this makes sense

> **Default: `False`**

Enables resolving imports, files, modules, and directories from outside of the repo path.

<Warning>
Turning this flag off may allow for bad actors to access files outside of the repo path! Use with caution!
</Warning>

## Flag: `ts_dependency_manager`
> **Default: `False`**

Expand Down
1 change: 1 addition & 0 deletions src/codegen/configs/models/codebase.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ def __init__(self, prefix: str = "CODEBASE", *args, **kwargs) -> None:
import_resolution_paths: list[str] = Field(default_factory=lambda: [])
import_resolution_overrides: dict[str, str] = Field(default_factory=lambda: {})
py_resolve_syspath: bool = False
allow_external: bool = False
ts_dependency_manager: bool = False
ts_language_engine: bool = False
v8_ts_engine: bool = False
Expand Down
4 changes: 2 additions & 2 deletions src/codegen/sdk/codebase/codebase_context.py
Original file line number Diff line number Diff line change
Expand Up @@ -381,7 +381,7 @@ def get_directory(self, directory_path: PathLike, create_on_missing: bool = Fals
"""
# If not part of repo path, return None
absolute_path = self.to_absolute(directory_path)
if not self.is_subdir(absolute_path):
if not self.is_subdir(absolute_path) and not self.config.allow_external:
assert False, f"Directory {absolute_path} is not part of repo path {self.repo_path}"
return None

Expand Down Expand Up @@ -611,7 +611,7 @@ def get_edges(self) -> list[tuple[NodeId, NodeId, EdgeType, Usage | None]]:
def get_file(self, file_path: os.PathLike, ignore_case: bool = False) -> SourceFile | None:
# If not part of repo path, return None
absolute_path = self.to_absolute(file_path)
if not self.is_subdir(absolute_path):
if not self.is_subdir(absolute_path) and not self.config.allow_external:
assert False, f"File {file_path} is not part of the repository path"

# Check if file exists in graph
Expand Down
26 changes: 26 additions & 0 deletions src/codegen/sdk/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -339,3 +339,29 @@ def is_minified_js(content):
except Exception as e:
print(f"Error analyzing content: {e}")
return False


@contextmanager
def use_cwd(path):
"""Context manager that temporarily changes the current working directory.

Args:
path (str): The directory path to change to.

Yields:
str: The new current working directory.

Example:
```python
with use_cwd("/path/to/directory"):
# Code here runs with the working directory set to '/path/to/directory'
...
# Working directory is restored to the original
```
"""
old_cwd = os.getcwd()
try:
os.chdir(path)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we use this as a pytest fixture?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, looking into it!

yield path
finally:
os.chdir(old_cwd)
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

from codegen.sdk.codebase.config import TestFlags
from codegen.sdk.codebase.factory.get_session import get_codebase_session
from codegen.sdk.utils import use_cwd

if TYPE_CHECKING:
from codegen.sdk.core.file import SourceFile
Expand Down Expand Up @@ -272,27 +273,31 @@ def func():
""",
},
) as codebase:
src_file: SourceFile = codebase.get_file("a/b/c/src.py")
consumer_file: SourceFile = codebase.get_file("consumer.py")
# Wrap CWD since we are using py_resolve_syspath
with use_cwd(tmpdir):
src_file: SourceFile = codebase.get_file("a/b/c/src.py")
consumer_file: SourceFile = codebase.get_file("consumer.py")

# Enable resolution via sys.path
codebase.ctx.config.py_resolve_syspath = True
# Enable resolution via sys.path
codebase.ctx.config.py_resolve_syspath = True
# Allow resolving files and modules outside of the repo path
codebase.ctx.config.allow_external = True

# =====[ Imports cannot be found without sys.path being set ]=====
assert len(consumer_file.imports) == 1
src_import: Import = consumer_file.imports[0]
src_import_resolution: ImportResolution = src_import.resolve_import()
assert src_import_resolution is None
# =====[ Imports cannot be found without sys.path being set ]=====
assert len(consumer_file.imports) == 1
src_import: Import = consumer_file.imports[0]
src_import_resolution: ImportResolution = src_import.resolve_import()
assert src_import_resolution is None

# Modify sys.path for this test only
monkeypatch.syspath_prepend("a")
# Modify sys.path for this test only
monkeypatch.syspath_prepend("a")

# =====[ Imports can be found with sys.path set and active ]=====
codebase.ctx.config.py_resolve_syspath = True
src_import_resolution = src_import.resolve_import()
assert src_import_resolution
assert src_import_resolution.from_file is src_file
assert src_import_resolution.imports_file is True
# =====[ Imports can be found with sys.path set and active ]=====
codebase.ctx.config.py_resolve_syspath = True
src_import_resolution = src_import.resolve_import()
assert src_import_resolution
assert src_import_resolution.from_file is src_file
assert src_import_resolution.imports_file is True


def test_import_resolution_file_custom_resolve_path(tmpdir: str) -> None:
Expand Down Expand Up @@ -366,28 +371,32 @@ def func():
""",
},
) as codebase:
src_file: SourceFile = codebase.get_file("a/b/c/src.py")
consumer_file: SourceFile = codebase.get_file("consumer.py")

# Ensure we don't have overrites and enable syspath resolution
codebase.ctx.config.import_resolution_paths = []
codebase.ctx.config.py_resolve_syspath = True

# =====[ Import with sys.path set can be found ]=====
assert len(consumer_file.imports) == 1
# Modify sys.path for this test only
monkeypatch.syspath_prepend("a")
src_import: Import = consumer_file.imports[0]
src_import_resolution = src_import.resolve_import()
assert src_import_resolution
assert src_import_resolution.from_file.file_path == "a/c/src.py"

# =====[ Imports can be found with custom resolve over sys.path ]=====
codebase.ctx.config.import_resolution_paths = ["a/b"]
src_import_resolution = src_import.resolve_import()
assert src_import_resolution
assert src_import_resolution.from_file is src_file
assert src_import_resolution.imports_file is True
# Wrap CWD since we are using py_resolve_syspath
with use_cwd(tmpdir):
src_file: SourceFile = codebase.get_file("a/b/c/src.py")
consumer_file: SourceFile = codebase.get_file("consumer.py")

# Ensure we don't have overrites and enable syspath resolution
codebase.ctx.config.import_resolution_paths = []
codebase.ctx.config.py_resolve_syspath = True
# Allow resolving files and modules outside of the repo path
codebase.ctx.config.allow_external = True

# =====[ Import with sys.path set can be found ]=====
assert len(consumer_file.imports) == 1
# Modify sys.path for this test only
monkeypatch.syspath_prepend("a")
src_import: Import = consumer_file.imports[0]
src_import_resolution = src_import.resolve_import()
assert src_import_resolution
assert src_import_resolution.from_file.file_path == "a/c/src.py"

# =====[ Imports can be found with custom resolve over sys.path ]=====
codebase.ctx.config.import_resolution_paths = ["a/b"]
src_import_resolution = src_import.resolve_import()
assert src_import_resolution
assert src_import_resolution.from_file is src_file
assert src_import_resolution.imports_file is True


def test_import_resolution_default_conflicts_overrite(tmpdir: str, monkeypatch) -> None:
Expand All @@ -412,34 +421,38 @@ def func():
""",
},
) as codebase:
src_file: SourceFile = codebase.get_file("a/src.py")
src_file_overrite: SourceFile = codebase.get_file("b/a/src.py")
consumer_file: SourceFile = codebase.get_file("consumer.py")

# Ensure we don't have overrites and enable syspath resolution
codebase.ctx.config.import_resolution_paths = []
codebase.ctx.config.py_resolve_syspath = True

# =====[ Default import works ]=====
assert len(consumer_file.imports) == 1
src_import: Import = consumer_file.imports[0]
src_import_resolution = src_import.resolve_import()
assert src_import_resolution
assert src_import_resolution.from_file is src_file

# =====[ Sys.path overrite has precedence ]=====
monkeypatch.syspath_prepend("b")
src_import_resolution = src_import.resolve_import()
assert src_import_resolution
assert src_import_resolution.from_file is not src_file
assert src_import_resolution.from_file is src_file_overrite

# =====[ Custom overrite has precedence ]=====
codebase.ctx.config.import_resolution_paths = ["b"]
src_import_resolution = src_import.resolve_import()
assert src_import_resolution
assert src_import_resolution.from_file is not src_file
assert src_import_resolution.from_file is src_file_overrite
# Wrap CWD since we are using py_resolve_syspath
with use_cwd(tmpdir):
src_file: SourceFile = codebase.get_file("a/src.py")
src_file_overrite: SourceFile = codebase.get_file("b/a/src.py")
consumer_file: SourceFile = codebase.get_file("consumer.py")

# Ensure we don't have overrites and enable syspath resolution
codebase.ctx.config.import_resolution_paths = []
codebase.ctx.config.py_resolve_syspath = True
# Allow resolving files and modules outside of the repo path
codebase.ctx.config.allow_external = True

# =====[ Default import works ]=====
assert len(consumer_file.imports) == 1
src_import: Import = consumer_file.imports[0]
src_import_resolution = src_import.resolve_import()
assert src_import_resolution
assert src_import_resolution.from_file is src_file

# =====[ Sys.path overrite has precedence ]=====
monkeypatch.syspath_prepend("b")
src_import_resolution = src_import.resolve_import()
assert src_import_resolution
assert src_import_resolution.from_file is not src_file
assert src_import_resolution.from_file is src_file_overrite

# =====[ Custom overrite has precedence ]=====
codebase.ctx.config.import_resolution_paths = ["b"]
src_import_resolution = src_import.resolve_import()
assert src_import_resolution
assert src_import_resolution.from_file is not src_file
assert src_import_resolution.from_file is src_file_overrite


def test_import_resolution_init_wildcard(tmpdir: str) -> None:
Expand Down
Loading