Fix codebase parsing for unicode filenames

src/codegen/git/repo_operator/repo_operator.py

@@ -1,3 +1,4 @@
+import codecs
 import fnmatch
 import glob
 import logging
@@ -70,7 +71,7 @@
         else:
             os.makedirs(self.repo_path, exist_ok=True)
             GitCLI.init(self.repo_path)
             self._local_git_repo = LocalGitRepo(repo_path=repo_config.repo_path)
 
             if repo_config.full_name is None:
                 repo_config.full_name = self._local_git_repo.full_name
@@ -142,7 +143,7 @@
         email_level = None
         levels = ["system", "global", "user", "repository"]
         for level in levels:
             with git_cli.config_reader(level) as reader:
                 if reader.has_option("user", "name") and not username:
                     username = reader.get("user", "name")
                     user_level = level
@@ -554,7 +555,7 @@
                     return content
             except UnicodeDecodeError:
                 print(f"Warning: Unable to decode file {file_path}. Skipping.")
                 return None
 
     def write_file(self, relpath: str, content: str) -> None:
         """Writes file content to disk"""
@@ -577,6 +578,25 @@
         if ignore_list:
             filepaths = [f for f in filepaths if not any(fnmatch.fnmatch(f, pattern) or f.startswith(pattern) for pattern in ignore_list)]
 
+        # Fix bug where unicode characters are not handled correctly
+        for i, filepath in enumerate(filepaths):
+            # Check if it is one of the broken cases
+            if filepath.startswith('"'):
+                # Step 1: Strip the quotes
+                filepath = filepath.strip('"').strip("'")
+
+                # Step 2: Convert the Python string to raw ASCII bytes (so \\ stays as two 0x5C).
+                raw_filepath = filepath.encode("ascii")
+
+                # Step 3: Use escape_decode to process backslash escapes like \346 -> 0xE6
+                decoded_filepath, _ = codecs.escape_decode(raw_filepath)
+
+                # Step 4: Decode those bytes as UTF-8 to get the actual Unicode text
+                filepath = decoded_filepath.decode("utf-8")
+
+                # Step 5: Replace the original filepath with the decoded filepath
+                filepaths[i] = filepath
+
         return filepaths
 
     # TODO: unify param naming i.e. subdirectories vs subdirs probably use subdirectories since that's in the DB
@@ -602,7 +622,7 @@
         filepaths = self.get_filepaths_for_repo(ignore_list)
         # Iterate through files and yield contents
         for rel_filepath in filepaths:
             rel_filepath: str
             filepath = os.path.join(self.repo_path, rel_filepath)
 
             # Filter by subdirectory (includes full filenames)
@@ -633,7 +653,7 @@
         list_files = []
 
         for rel_filepath in self.git_cli.git.ls_files().split("\n"):
             rel_filepath: str
             if subdirs and not any(d in rel_filepath for d in subdirs):
                 continue
             if extensions is None or any(rel_filepath.endswith(e) for e in extensions):
@@ -657,7 +677,7 @@
 
     def get_modified_files_in_last_n_days(self, days: int = 1) -> tuple[list[str], list[str]]:
         """Returns a list of files modified and deleted in the last n days"""
         modified_files = []
         deleted_files = []
         allowed_extensions = [".py"]
 
@@ -673,9 +693,9 @@
                     if file in modified_files:
                         modified_files.remove(file)
                 else:
                     if file not in modified_files and file[-3:] in allowed_extensions:
                         modified_files.append(file)
         return modified_files, deleted_files
 
     @cached_property
     def base_url(self) -> str | None:
@@ -696,7 +716,7 @@
 
     def get_pr_data(self, pr_number: int) -> dict:
         """Returns the data associated with a PR"""
         return self.remote_git_repo.get_pr_data(pr_number)
 
     def create_pr_comment(self, pr_number: int, body: str) -> None:
         """Create a general comment on a pull request.
@@ -739,7 +759,7 @@
                     body=body,
                     commit=commit,
                     path=path,
                     line=line,
                     side=side,
                     start_line=start_line,
                 )

tests/unit/codegen/sdk/core/test_directory.py

@@ -7,8 +7,10 @@
 
 from codegen.sdk.codebase.codebase_context import CodebaseContext
 from codegen.sdk.codebase.config import CodebaseConfig
+from codegen.sdk.codebase.factory.get_session import get_codebase_session
 from codegen.sdk.core.directory import Directory
 from codegen.sdk.core.file import File
+from codegen.shared.enums.programming_language import ProgrammingLanguage
 
 
 @pytest.fixture
@@ -220,3 +222,15 @@ def test_get_set_delete_item(mock_directory):
 
     with pytest.raises(KeyError, match="subdir_2"):
         del mock_directory["subdir_2"]
+
+
+def test_unicode_in_filename(tmpdir) -> None:
+    with get_codebase_session(
+        tmpdir=tmpdir,
+        files={"ascii.py": "print('Hello, world!')", "test/我很喜欢冰激淋/test-file 12'3_🍦.py": "print('Hello, world!')"},
+        programming_language=ProgrammingLanguage.PYTHON,
+        verify_output=True,
+    ) as codebase:
+        file = codebase.get_file("test/我很喜欢冰激淋/test-file 12'3_🍦.py")
+        assert file is not None
+        assert file.content == "print('Hello, world!')"