Sepcify the encoding (#2515)

Author: FFY00

docs/changelog/2515.bugfix.rst

@@ -0,0 +1 @@
+Do not assume the default encoding.

src/virtualenv/app_data/via_disk_folder.py

@@ -125,7 +125,7 @@ def exists(self):
     def read(self):
         data, bad_format = None, False
         try:
-            data = json.loads(self.file.read_text())
+            data = json.loads(self.file.read_text(encoding="utf-8"))
             logging.debug(f"got {self.msg} from %s", *self.msg_args)
             return data
         except ValueError:
@@ -151,7 +151,7 @@ def locked(self):
     def write(self, content):
         folder = self.file.parent
         folder.mkdir(parents=True, exist_ok=True)
-        self.file.write_text(json.dumps(content, sort_keys=True, indent=2))
+        self.file.write_text(json.dumps(content, sort_keys=True, indent=2), encoding="utf-8")
         logging.debug(f"wrote {self.msg} at %s", *self.msg_args)
 
 

src/virtualenv/config/ini.py

@@ -44,7 +44,7 @@ def __init__(self, env=None):
             logging.error("failed to read config file %s because %r", config_file, exception)
 
     def _load(self):
-        with self.config_file.open("rt") as file_handler:
+        with self.config_file.open("rt", encoding="utf-8") as file_handler:
             return self.config_parser.read_file(file_handler)
 
     def get(self, key, as_type):

src/virtualenv/create/creator.py

@@ -163,7 +163,7 @@ def setup_ignore_vcs(self):
         # mark this folder to be ignored by VCS, handle https://www.python.org/dev/peps/pep-0610/#registered-vcs
         git_ignore = self.dest / ".gitignore"
         if not git_ignore.exists():
-            git_ignore.write_text("# created by virtualenv automatically\n*\n")
+            git_ignore.write_text("# created by virtualenv automatically\n*\n", encoding="utf-8")
         # Mercurial - does not support the .hgignore file inside a subdirectory directly, but only if included via the
         # subinclude directive from root, at which point on might as well ignore the directory itself, see
         # https://www.selenic.com/mercurial/hgignore.5.html for more details

src/virtualenv/create/via_global_ref/api.py

@@ -87,15 +87,15 @@ def install_patch(self):
         if text:
             pth = self.purelib / "_virtualenv.pth"
             logging.debug("create virtualenv import hook file %s", pth)
-            pth.write_text("import _virtualenv")
+            pth.write_text("import _virtualenv", encoding="utf-8")
             dest_path = self.purelib / "_virtualenv.py"
             logging.debug("create %s", dest_path)
-            dest_path.write_text(text)
+            dest_path.write_text(text, encoding="utf-8")
 
     def env_patch_text(self):
         """Patch the distutils package to not be derailed by its configuration files"""
         with self.app_data.ensure_extracted(Path(__file__).parent / "_virtualenv.py") as resolved_path:
-            text = resolved_path.read_text()
+            text = resolved_path.read_text(encoding="utf-8")
             return text.replace('"__SCRIPT_DIR__"', repr(os.path.relpath(str(self.script_dir), str(self.purelib))))
 
     def _args(self):

src/virtualenv/create/via_global_ref/builtin/python2/python2.py

@@ -31,7 +31,7 @@ def create(self):
         if IS_ZIPAPP:
             custom_site_text = read_from_zipapp(custom_site)
         else:
-            custom_site_text = custom_site.read_text()
+            custom_site_text = custom_site.read_text(encoding="utf-8")
         expected = json.dumps([os.path.relpath(str(i), str(site_py)) for i in self.libs])
 
         custom_site_text = custom_site_text.replace("___EXPECTED_SITE_PACKAGES___", expected)
@@ -42,7 +42,7 @@ def create(self):
         skip_rewrite = os.linesep.join(f"            {i}" for i in self.skip_rewrite.splitlines()).lstrip()
         custom_site_text = custom_site_text.replace("# ___SKIP_REWRITE____", skip_rewrite)
 
-        site_py.write_text(custom_site_text)
+        site_py.write_text(custom_site_text, encoding="utf-8")
 
     @property
     def reload_code(self):

src/virtualenv/discovery/cached_py_info.py

@@ -114,6 +114,7 @@ def _run_subprocess(cls, exe, app_data, env):
                 stderr=subprocess.PIPE,
                 stdout=subprocess.PIPE,
                 env=env,
+                encoding="utf-8",
             )
             out, err = process.communicate()
             code = process.returncode

src/virtualenv/seed/embed/via_app_data/pip_install/base.py

@@ -72,11 +72,11 @@ def _records_text(self, files):
     def _generate_new_files(self):
         new_files = set()
         installer = self._dist_info / "INSTALLER"
-        installer.write_text("pip\n")
+        installer.write_text("pip\n", encoding="utf-8")
         new_files.add(installer)
         # inject a no-op root element, as workaround for bug in https://github.com/pypa/pip/issues/7226
         marker = self._image_dir / f"{self._dist_info.stem}.virtualenv"
-        marker.write_text("")
+        marker.write_text("", encoding="utf-8")
         new_files.add(marker)
         folder = mkdtemp()
         try:
@@ -120,7 +120,7 @@ def _console_scripts(self):
             entry_points = self._dist_info / "entry_points.txt"
             if entry_points.exists():
                 parser = ConfigParser()
-                with entry_points.open() as file_handler:
+                with entry_points.open(encoding="utf-8") as file_handler:
                     parser.read_file(file_handler)
                 if "console_scripts" in parser.sections():
                     for name, value in parser.items("console_scripts"):
@@ -152,11 +152,17 @@ def _uninstall_dist(dist):
         logging.debug("uninstall existing distribution %s from %s", dist.stem, dist_base)
 
         top_txt = dist / "top_level.txt"  # add top level packages at folder level
-        paths = {dist.parent / i.strip() for i in top_txt.read_text().splitlines()} if top_txt.exists() else set()
+        paths = (
+            {dist.parent / i.strip() for i in top_txt.read_text(encoding="utf-8").splitlines()}
+            if top_txt.exists()
+            else set()
+        )
         paths.add(dist)  # add the dist-info folder itself
 
         base_dirs, record = paths.copy(), dist / "RECORD"  # collect entries in record that we did not register yet
-        for name in (i.split(",")[0] for i in record.read_text().splitlines()) if record.exists() else ():
+        for name in (
+            (i.split(",")[0] for i in record.read_text(encoding="utf-8").splitlines()) if record.exists() else ()
+        ):
             path = dist_base / name
             if not any(p in base_dirs for p in path.parents):  # only add if not already added as a base dir
                 paths.add(path)

src/virtualenv/seed/wheels/acquire.py

@@ -60,7 +60,7 @@ def download_wheel(distribution, version_spec, for_py_version, search_dirs, app_
     ]
     # pip has no interface in python - must be a new sub-process
     env = pip_wheel_env_run(search_dirs, app_data, env)
-    process = Popen(cmd, env=env, stdout=PIPE, stderr=PIPE, universal_newlines=True)
+    process = Popen(cmd, env=env, stdout=PIPE, stderr=PIPE, universal_newlines=True, encoding="utf-8")
     out, err = process.communicate()
     if process.returncode != 0:
         kwargs = {"output": out, "stderr": err}

src/virtualenv/util/subprocess/__init__.py

@@ -11,6 +11,7 @@ def run_cmd(cmd):
             stdin=subprocess.PIPE,
             stderr=subprocess.PIPE,
             stdout=subprocess.PIPE,
+            encoding="utf-8",
         )
         out, err = process.communicate()  # input disabled
         code = process.returncode

tests/unit/activation/conftest.py

@@ -36,6 +36,7 @@ def get_version(self, raise_on_fail):
                     universal_newlines=True,
                     stdout=subprocess.PIPE,
                     stderr=subprocess.PIPE,
+                    encoding="utf-8",
                 )
                 out, err = process.communicate()
                 result = out if out else err
@@ -220,7 +221,7 @@ def activation_python(request, tmp_path_factory, special_char_name, current_fast
         cmd += ["--prompt", special_char_name]
     session = cli_run(cmd)
     pydoc_test = session.creator.purelib / "pydoc_test.py"
-    pydoc_test.write_text('"""This is pydoc_test.py"""')
+    pydoc_test.write_text('"""This is pydoc_test.py"""', encoding="utf-8")
     return session
 
 

tests/unit/activation/test_activate_this.py

@@ -21,6 +21,6 @@ def test_python_activator_cross(session_app_data, cross_python, special_name_dir
     results = activator.generate(session.creator)
     assert len(results) == 1
     result = results[0]
-    content = result.read_text()
+    content = result.read_text(encoding="utf-8")
     # check that the repr strings have been correctly stripped
     assert "\"'" not in content

tests/unit/activation/test_batch.py

@@ -8,7 +8,7 @@
 @pytest.mark.usefixtures("activation_python")
 def test_batch(activation_tester_class, activation_tester, tmp_path):
     version_script = tmp_path / "version.bat"
-    version_script.write_text("ver")
+    version_script.write_text("ver", encoding="utf-8")
 
     class Batch(activation_tester_class):
         def __init__(self, session):

tests/unit/activation/test_fish.py

@@ -9,7 +9,7 @@ def test_fish(activation_tester_class, activation_tester, monkeypatch, tmp_path)
     monkeypatch.setenv("HOME", str(tmp_path))
     fish_conf_dir = tmp_path / ".config" / "fish"
     fish_conf_dir.mkdir(parents=True)
-    (fish_conf_dir / "config.fish").write_text("")
+    (fish_conf_dir / "config.fish").write_text("", encoding="utf-8")
 
     class Fish(activation_tester_class):
         def __init__(self, session):

tests/unit/activation/test_python_activator.py

@@ -44,7 +44,7 @@ def print_r(value):
 
             file_at = {str(activate_script)!r}
             # CPython 2 requires non-ascii path open to be unicode
-            with open(file_at, "r") as file_handler:
+            with open(file_at, "r", encoding='utf-8') as file_handler:
                 content = file_handler.read()
             exec(content, {{"__file__": file_at}})
 

tests/unit/config/test___main__.py

@@ -9,7 +9,12 @@
 
 
 def test_main():
-    process = Popen([sys.executable, "-m", "virtualenv", "--help"], universal_newlines=True, stdout=PIPE)
+    process = Popen(
+        [sys.executable, "-m", "virtualenv", "--help"],
+        universal_newlines=True,
+        stdout=PIPE,
+        encoding="utf-8",
+    )
     out, _ = process.communicate()
     assert not process.returncode
     assert out
@@ -92,6 +97,7 @@ def test_session_report_subprocess(tmp_path):
     out = check_output(
         [sys.executable, "-m", "virtualenv", str(tmp_path), "--activators", "powershell", "--without-pip"],
         text=True,
+        encoding="utf-8",
     )
     lines = out.split("\n")
     regexes = [

tests/unit/config/test_env_var.py

@@ -12,7 +12,7 @@
 def _empty_conf(tmp_path, monkeypatch):
     conf = tmp_path / "conf.ini"
     monkeypatch.setenv(IniConfig.VIRTUALENV_CONFIG_FILE_ENV_VAR, str(conf))
-    conf.write_text("[virtualenv]")
+    conf.write_text("[virtualenv]", encoding="utf-8")
 
 
 @pytest.mark.usefixtures("_empty_conf")

tests/unit/config/test_ini.py

@@ -16,6 +16,7 @@ def test_ini_can_be_overwritten_by_flag(tmp_path, monkeypatch):
         copies = True
         """,
         ),
+        encoding="utf-8",
     )
     monkeypatch.setenv("VIRTUALENV_CONFIG_FILE", str(custom_ini))
 

tests/unit/create/test_creator.py

@@ -53,7 +53,7 @@ def _non_success_exit_code(capsys, target):
 
 def test_destination_exists_file(tmp_path, capsys):
     target = tmp_path / "out"
-    target.write_text("")
+    target.write_text("", encoding="utf-8")
     err = _non_success_exit_code(capsys, str(target))
     msg = f"the destination {str(target)} already exists and is a file"
     assert msg in err, err
@@ -230,15 +230,15 @@ def list_to_str(iterable):
         make_file = debug["makefile_filename"]
         assert os.path.exists(make_file)
 
-    git_ignore = (dest / ".gitignore").read_text()
+    git_ignore = (dest / ".gitignore").read_text(encoding="utf-8")
     assert git_ignore.splitlines() == ["# created by virtualenv automatically", "*"]
 
 
 def test_create_vcs_ignore_exists(tmp_path):
     git_ignore = tmp_path / ".gitignore"
-    git_ignore.write_text("magic")
+    git_ignore.write_text("magic", encoding="utf-8")
     cli_run([str(tmp_path), "--without-pip", "--activators", ""])
-    assert git_ignore.read_text() == "magic"
+    assert git_ignore.read_text(encoding="utf-8") == "magic"
 
 
 def test_create_vcs_ignore_override(tmp_path):
@@ -249,9 +249,9 @@ def test_create_vcs_ignore_override(tmp_path):
 
 def test_create_vcs_ignore_exists_override(tmp_path):
     git_ignore = tmp_path / ".gitignore"
-    git_ignore.write_text("magic")
+    git_ignore.write_text("magic", encoding="utf-8")
     cli_run([str(tmp_path), "--without-pip", "--no-vcs-ignore", "--activators", ""])
-    assert git_ignore.read_text() == "magic"
+    assert git_ignore.read_text(encoding="utf-8") == "magic"
 
 
 @pytest.mark.skipif(not CURRENT.has_venv, reason="requires interpreter with venv")
@@ -268,7 +268,7 @@ def _session_via_cli(args, options=None, setup_logging=True, env=None):
     mocker.patch("virtualenv.run.session_via_cli", side_effect=_session_via_cli)
     before = tmp_path.stat().st_mode
     cfg_path = tmp_path / "pyvenv.cfg"
-    cfg_path.write_text("")
+    cfg_path.write_text("", encoding="utf-8")
     cfg = str(cfg_path)
     try:
         os.chmod(cfg, stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH)
@@ -293,7 +293,7 @@ def test_create_clear_resets(tmp_path, creator, clear, caplog):
     cmd = [str(tmp_path), "--seeder", "app-data", "--without-pip", "--creator", creator, "-vvv"]
     cli_run(cmd)
 
-    marker.write_text("")  # if we a marker file this should be gone on a clear run, remain otherwise
+    marker.write_text("", encoding="utf-8")  # if we a marker file this should be gone on a clear run, remain otherwise
     assert marker.exists()
 
     cli_run(cmd + (["--clear"] if clear else []))
@@ -430,7 +430,7 @@ def test_create_distutils_cfg(creator, tmp_path, monkeypatch):
             install_data={tmp_path}{os.sep}data
             """,
     )
-    setup_cfg.write_text(setup_cfg.read_text() + conf)
+    setup_cfg.write_text(setup_cfg.read_text(encoding="utf-8") + conf, encoding="utf-8")
 
     monkeypatch.chdir(dest)  # distutils will read the setup.cfg from the cwd, so change to that
 
@@ -509,7 +509,7 @@ def test_pth_in_site_vs_python_path(tmp_path):
     session = cli_run([str(tmp_path)])
     site_packages = str(session.creator.purelib)
     # install test.pth that sets sys.testpth='ok'
-    with open(os.path.join(site_packages, "test.pth"), "w") as f:
+    with open(os.path.join(site_packages, "test.pth"), "w", encoding="utf-8") as f:
         f.write('import sys; sys.testpth="ok"\n')
     # verify that test.pth is activated when interpreter is run
     out = subprocess.check_output(
@@ -595,7 +595,10 @@ def test_debug_bad_virtualenv(tmp_path):
     result = cli_run(cmd)
     # if the site.py is removed/altered the debug should fail as no one is around to fix the paths
     cust = result.creator.purelib / "_a.pth"
-    cust.write_text('import sys; sys.stdout.write("std-out"); sys.stderr.write("std-err"); raise SystemExit(1)')
+    cust.write_text(
+        'import sys; sys.stdout.write("std-out"); sys.stderr.write("std-err"); raise SystemExit(1)',
+        encoding="utf-8",
+    )
     debug_info = result.creator.debug
     assert debug_info["returncode"] == 1
     assert "std-err" in debug_info["err"]

tests/unit/create/via_global_ref/builtin/testing/py_info.py

@@ -15,5 +15,5 @@ def fixture_file(fixture_name):
 
 
 def read_fixture(fixture_name):
-    fixture_json = fixture_file(fixture_name).read_text()
+    fixture_json = fixture_file(fixture_name).read_text(encoding="utf-8")
     return PythonInfo._from_json(fixture_json)

tests/unit/create/via_global_ref/test_build_c_ext.py

@@ -56,6 +56,7 @@ def test_can_build_c_extensions(creator, tmp_path, coverage_env):
         [str(session.creator.exe), "-c", "import greet; greet.greet('World')"],
         universal_newlines=True,
         stdout=subprocess.PIPE,
+        encoding="utf-8",
     )
     out, _ = process.communicate()
     assert process.returncode == 0

tests/unit/discovery/py_info/test_py_info.py

@@ -156,7 +156,7 @@ def test_py_info_cached_symlink(mocker, tmp_path, session_app_data):
     new_exe.symlink_to(sys.executable)
     pyvenv = Path(sys.executable).parents[1] / "pyvenv.cfg"
     if pyvenv.exists():
-        (tmp_path / pyvenv.name).write_text(pyvenv.read_text())
+        (tmp_path / pyvenv.name).write_text(pyvenv.read_text(encoding="utf-8"), encoding="utf-8")
     new_exe_str = str(new_exe)
     second_result = PythonInfo.from_exe(new_exe_str, session_app_data)
     assert second_result.executable == new_exe_str
@@ -211,7 +211,7 @@ def _make_py_info(of):
     selected = None
     for pos, i in enumerate(discovered):
         path = tmp_path / str(pos)
-        path.write_text("")
+        path.write_text("", encoding="utf-8")
         py_info = _make_py_info(i)
         py_info.system_executable = CURRENT.system_executable
         py_info.executable = CURRENT.system_executable
@@ -259,7 +259,7 @@ def test_py_info_ignores_distutils_config(monkeypatch, tmp_path):
     install_scripts={tmp_path}{os.sep}scripts
     install_data={tmp_path}{os.sep}data
     """
-    (tmp_path / "setup.cfg").write_text(dedent(raw))
+    (tmp_path / "setup.cfg").write_text(dedent(raw), encoding="utf-8")
     monkeypatch.chdir(tmp_path)
     py_info = PythonInfo.from_exe(sys.executable)
     distutils = py_info.distutils_install

tests/unit/discovery/py_info/test_py_info_exe_based_of.py

@@ -36,7 +36,7 @@ def test_discover_ok(tmp_path, suffix, impl, version, arch, into, caplog, sessio
     os.symlink(CURRENT.executable, str(dest))
     pyvenv = Path(CURRENT.executable).parents[1] / "pyvenv.cfg"
     if pyvenv.exists():
-        (folder / pyvenv.name).write_text(pyvenv.read_text())
+        (folder / pyvenv.name).write_text(pyvenv.read_text(encoding="utf-8"), encoding="utf-8")
     inside_folder = str(tmp_path)
     base = CURRENT.discover_exe(session_app_data, inside_folder)
     found = base.executable

tox.ini

@@ -34,6 +34,7 @@ set_env =
     _COVERAGE_SRC = {envsitepackagesdir}/virtualenv
     COVERAGE_FILE = {toxworkdir}/.coverage.{envname}
     COVERAGE_PROCESS_START = {toxinidir}/pyproject.toml
+    PYTHONWARNDEFAULTENCODING = 1
 wheel_build_env = .pkg
 
 [testenv:fix]