bpo-41043: Escape literal part of the path for glob(). (GH-20994)

Author: serhiy-storchaka

Lib/distutils/command/build_py.py

@@ -5,7 +5,7 @@
 import os
 import importlib.util
 import sys
-from glob import glob
+import glob
 
 from distutils.core import Command
 from distutils.errors import *
@@ -125,7 +125,7 @@ def find_data_files(self, package, src_dir):
         files = []
         for pattern in globs:
             # Each pattern has to be converted to a platform-specific path
-            filelist = glob(os.path.join(src_dir, convert_path(pattern)))
+            filelist = glob.glob(os.path.join(glob.escape(src_dir), convert_path(pattern)))
             # Files that match more than one pattern are only added once
             files.extend([fn for fn in filelist if fn not in files
                 and os.path.isfile(fn)])
@@ -216,7 +216,7 @@ def check_module(self, module, module_file):
 
     def find_package_modules(self, package, package_dir):
         self.check_package(package, package_dir)
-        module_files = glob(os.path.join(package_dir, "*.py"))
+        module_files = glob.glob(os.path.join(glob.escape(package_dir), "*.py"))
         modules = []
         setup_script = os.path.abspath(self.distribution.script_name)
 

Lib/idlelib/tree.py

@@ -38,7 +38,7 @@ def listicons(icondir=ICONDIR):
     """Utility to display the available icons."""
     root = Tk()
     import glob
-    list = glob.glob(os.path.join(icondir, "*.gif"))
+    list = glob.glob(os.path.join(glob.escape(icondir), "*.gif"))
     list.sort()
     images = []
     row = column = 0

Lib/imghdr.py

@@ -152,7 +152,7 @@ def testall(list, recursive, toplevel):
             if recursive or toplevel:
                 print('recursing down:')
                 import glob
-                names = glob.glob(os.path.join(filename, '*'))
+                names = glob.glob(os.path.join(glob.escape(filename), '*'))
                 testall(names, recursive, 0)
             else:
                 print('*** directory (use -r) ***')

Lib/pdb.py

@@ -473,7 +473,7 @@ def _complete_location(self, text, line, begidx, endidx):
         except Exception:
             ret = []
         # Then, try to complete file names as well.
-        globs = glob.glob(text + '*')
+        globs = glob.glob(glob.escape(text) + '*')
         for fn in globs:
             if os.path.isdir(fn):
                 ret.append(fn + '/')

Lib/sndhdr.py

@@ -241,7 +241,7 @@ def testall(list, recursive, toplevel):
             if recursive or toplevel:
                 print('recursing down:')
                 import glob
-                names = glob.glob(os.path.join(filename, '*'))
+                names = glob.glob(os.path.join(glob.escape(filename), '*'))
                 testall(names, recursive, 0)
             else:
                 print('*** directory (use -r) ***')

Lib/test/_test_multiprocessing.py

@@ -4260,7 +4260,7 @@ class _TestImportStar(unittest.TestCase):
     def get_module_names(self):
         import glob
         folder = os.path.dirname(multiprocessing.__file__)
-        pattern = os.path.join(folder, '*.py')
+        pattern = os.path.join(glob.escape(folder), '*.py')
         files = glob.glob(pattern)
         modules = [os.path.splitext(os.path.split(f)[1])[0] for f in files]
         modules = ['multiprocessing.' + m for m in modules]

Lib/test/libregrtest/main.py

@@ -602,7 +602,7 @@ def create_temp_dir(self):
     def cleanup(self):
         import glob
 
-        path = os.path.join(self.tmp_dir, 'test_python_*')
+        path = os.path.join(glob.escape(self.tmp_dir), 'test_python_*')
         print("Cleanup %s directory" % self.tmp_dir)
         for name in glob.glob(path):
             if os.path.isdir(name):

Lib/test/support/__init__.py

@@ -1345,7 +1345,7 @@ def _platform_specific(self):
                 dll,
                 os.path.join(dest_dir, os.path.basename(dll))
             ))
-            for runtime in glob.glob(os.path.join(src_dir, "vcruntime*.dll")):
+            for runtime in glob.glob(os.path.join(glob.escape(src_dir), "vcruntime*.dll")):
                 self._also_link.append((
                     runtime,
                     os.path.join(dest_dir, os.path.basename(runtime))

Lib/test/test_bz2.py

@@ -70,7 +70,7 @@ class BaseTest(unittest.TestCase):
     # simply use the bigger test data for all tests.
     test_size = 0
     BIG_TEXT = bytearray(128*1024)
-    for fname in glob.glob(os.path.join(os.path.dirname(__file__), '*.py')):
+    for fname in glob.glob(os.path.join(glob.escape(os.path.dirname(__file__)), '*.py')):
         with open(fname, 'rb') as fh:
             test_size += fh.readinto(memoryview(BIG_TEXT)[test_size:])
         if test_size > 128*1024:

Lib/test/test_crashers.py

@@ -11,7 +11,7 @@
 from test.support.script_helper import assert_python_failure
 
 CRASHER_DIR = os.path.join(os.path.dirname(__file__), "crashers")
-CRASHER_FILES = os.path.join(CRASHER_DIR, "*.py")
+CRASHER_FILES = os.path.join(glob.escape(CRASHER_DIR), "*.py")
 
 infinite_loops = ["infinite_loop_re.py", "nasty_eq_vs_dict.py"]
 

Lib/test/test_dbm.py

@@ -33,7 +33,7 @@ def dbm_iterator():
 def delete_files():
     # we don't know the precise name the underlying database uses
     # so we use glob to locate all names
-    for f in glob.glob(_fname + "*"):
+    for f in glob.glob(glob.escape(_fname) + "*"):
         test.support.unlink(f)
 
 

Lib/test/test_import/__init__.py

@@ -486,7 +486,7 @@ def test_dll_dependency_import(self):
             pyexe = os.path.join(tmp, os.path.basename(sys.executable))
             shutil.copy(sys.executable, pyexe)
             shutil.copy(dllname, tmp)
-            for f in glob.glob(os.path.join(sys.prefix, "vcruntime*.dll")):
+            for f in glob.glob(os.path.join(glob.escape(sys.prefix), "vcruntime*.dll")):
                 shutil.copy(f, tmp)
 
             shutil.copy(pydname, tmp2)

Lib/test/test_mailbox.py

@@ -979,7 +979,7 @@ def tearDown(self):
         super().tearDown()
         self._box.close()
         self._delete_recursively(self._path)
-        for lock_remnant in glob.glob(self._path + '.*'):
+        for lock_remnant in glob.glob(glob.escape(self._path) + '.*'):
             support.unlink(lock_remnant)
 
     def assertMailboxEmpty(self):
@@ -1311,7 +1311,7 @@ def tearDown(self):
         super().tearDown()
         self._box.close()
         self._delete_recursively(self._path)
-        for lock_remnant in glob.glob(self._path + '.*'):
+        for lock_remnant in glob.glob(glob.escape(self._path) + '.*'):
             support.unlink(lock_remnant)
 
     def test_labels(self):

Lib/test/test_regrtest.py

@@ -556,7 +556,7 @@ def test_finds_expected_number_of_tests(self):
         args = ['-Wd', '-E', '-bb', '-m', 'test.regrtest', '--list-tests']
         output = self.run_python(args)
         rough_number_of_tests_found = len(output.splitlines())
-        actual_testsuite_glob = os.path.join(os.path.dirname(__file__),
+        actual_testsuite_glob = os.path.join(glob.escape(os.path.dirname(__file__)),
                                              'test*.py')
         rough_counted_test_py_files = len(glob.glob(actual_testsuite_glob))
         # We're not trying to duplicate test finding logic in here,

Lib/test/test_site.py

@@ -543,7 +543,7 @@ def test_startup_imports(self):
         # found in sys.path (see site.addpackage()). Skip the test if at least
         # one .pth file is found.
         for path in isolated_paths:
-            pth_files = glob.glob(os.path.join(path, "*.pth"))
+            pth_files = glob.glob(os.path.join(glob.escape(path), "*.pth"))
             if pth_files:
                 self.skipTest(f"found {len(pth_files)} .pth files in: {path}")
 

Lib/test/test_tokenize.py

@@ -1605,7 +1605,7 @@ def test_random_files(self):
         import glob, random
         fn = support.findfile("tokenize_tests.txt")
         tempdir = os.path.dirname(fn) or os.curdir
-        testfiles = glob.glob(os.path.join(tempdir, "test*.py"))
+        testfiles = glob.glob(os.path.join(glob.escape(tempdir), "test*.py"))
 
         # Tokenize is broken on test_pep3131.py because regular expressions are
         # broken on the obscure unicode identifiers in it. *sigh*

Lib/test/test_unicode_file.py

@@ -41,7 +41,7 @@ def _do_single(self, filename):
         self._do_copyish(filename, filename)
         # Filename should appear in glob output
         self.assertTrue(
-            os.path.abspath(filename)==os.path.abspath(glob.glob(filename)[0]))
+            os.path.abspath(filename)==os.path.abspath(glob.glob(glob.escape(filename))[0]))
         # basename should appear in listdir.
         path, base = os.path.split(os.path.abspath(filename))
         file_list = os.listdir(path)

Lib/webbrowser.py

@@ -413,7 +413,7 @@ def _find_grail_rc(self):
         tempdir = os.path.join(tempfile.gettempdir(),
                                ".grail-unix")
         user = pwd.getpwuid(os.getuid())[0]
-        filename = os.path.join(tempdir, user + "-*")
+        filename = os.path.join(glob.escape(tempdir), glob.escape(user) + "-*")
         maybes = glob.glob(filename)
         if not maybes:
             return None

Misc/NEWS.d/next/Library/2020-06-20-00-19-30.bpo-41043.p-Pk-H.rst

@@ -0,0 +1,2 @@
+Fixed the use of :func:`~glob.glob` in the stdlib: literal part of the path
+is now always correctly escaped.

Tools/c-analyzer/c_analyzer/common/files.py

@@ -41,6 +41,8 @@ def walk_tree(root, *,
 def glob_tree(root, *,
               suffix=None,
               _glob=glob.iglob,
+              _escape=glob.escape,
+              _join=os.path.join,
               ):
     """Yield each file in the tree under the given directory name.
 
@@ -51,9 +53,9 @@ def glob_tree(root, *,
     if not isinstance(suffix, str):
         raise ValueError('suffix must be a string')
 
-    for filename in _glob(f'{root}/*{suffix}'):
+    for filename in _glob(_join(_escape(root), f'*{suffix}')):
         yield filename
-    for filename in _glob(f'{root}/**/*{suffix}'):
+    for filename in _glob(_join(_escape(root), f'**/*{suffix}')):
         yield filename
 
 

Tools/c-analyzer/check-c-globals.py

@@ -37,7 +37,9 @@
 def find_capi_vars(root):
     capi_vars = {}
     for dirname in SOURCE_DIRS:
-        for filename in glob.glob(os.path.join(ROOT_DIR, dirname, '**/*.[hc]'),
+        for filename in glob.glob(os.path.join(
+                                  glob.escape(os.path.join(ROOT_DIR, dirname)),
+                                  '**/*.[hc]'),
                                   recursive=True):
             with open(filename) as file:
                 for name in _find_capi_vars(file):

Tools/peg_generator/scripts/test_parse_directory.py

@@ -7,7 +7,7 @@
 import time
 import traceback
 import tokenize
-from glob import glob
+from glob import glob, escape
 from pathlib import PurePath
 
 from typing import List, Optional, Any, Tuple
@@ -109,7 +109,7 @@ def parse_directory(directory: str, verbose: bool, excluded_files: List[str], sh
     files = []
     total_seconds = 0
 
-    for file in sorted(glob(f"{directory}/**/*.py", recursive=True)):
+    for file in sorted(glob(os.path.join(escape(directory), f"**/*.py"), recursive=True)):
         # Only attempt to parse Python files and files that are not excluded
         if any(PurePath(file).match(pattern) for pattern in excluded_files):
             continue

Tools/ssl/make_ssl_data.py

@@ -39,7 +39,7 @@ def parse_error_codes(h_file, prefix, libcode):
     f = sys.stdout if use_stdout else open(outfile, "w")
     # mnemonic -> (library code, error prefix, header file)
     error_libraries = {}
-    for error_header in glob.glob(os.path.join(openssl_inc, 'include/openssl/*err.h')):
+    for error_header in glob.glob(os.path.join(glob.escape(openssl_inc), 'include/openssl/*err.h')):
         base = os.path.basename(error_header)
         if base in ('buffererr.h', 'objectserr.h', 'storeerr.h'):
             # Deprecated in 3.0.

setup.py

@@ -8,7 +8,7 @@
 import re
 import sys
 import sysconfig
-from glob import glob
+from glob import glob, escape
 
 
 try:
@@ -401,7 +401,7 @@ def update_sources_depends(self):
 
         # Python header files
         headers = [sysconfig.get_config_h_filename()]
-        headers += glob(os.path.join(sysconfig.get_path('include'), "*.h"))
+        headers += glob(os.path.join(escape(sysconfig.get_path('include')), "*.h"))
 
         for ext in self.extensions:
             ext.sources = [ find_module_file(filename, moddirlist)
@@ -2431,7 +2431,7 @@ def detect_hash_builtins(self):
 
         if "blake2" in configured:
             blake2_deps = glob(
-                os.path.join(self.srcdir, 'Modules/_blake2/impl/*')
+                os.path.join(escape(self.srcdir), 'Modules/_blake2/impl/*')
             )
             blake2_deps.append('hashlib.h')
             self.add(Extension(
@@ -2446,7 +2446,7 @@ def detect_hash_builtins(self):
 
         if "sha3" in configured:
             sha3_deps = glob(
-                os.path.join(self.srcdir, 'Modules/_sha3/kcp/*')
+                os.path.join(escape(self.srcdir), 'Modules/_sha3/kcp/*')
             )
             sha3_deps.append('hashlib.h')
             self.add(Extension(