Merge pull request #537 from python/main

Sync Fork from Upstream Repo

Author: sthagen

Doc/distributing/index.rst

@@ -131,11 +131,11 @@ involved in creating and publishing a project:
 * `The .pypirc file`_
 
 .. _Project structure: \
-    https://packaging.python.org/tutorials/distributing-packages/
+    https://packaging.python.org/tutorials/packaging-projects/#packaging-python-projects
 .. _Building and packaging the project: \
-   https://packaging.python.org/tutorials/distributing-packages/#packaging-your-project
+   https://packaging.python.org/tutorials/packaging-projects/#creating-the-package-files
 .. _Uploading the project to the Python Packaging Index: \
-   https://packaging.python.org/tutorials/distributing-packages/#uploading-your-project-to-pypi
+   https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives
 .. _The .pypirc file: \
    https://packaging.python.org/specifications/pypirc/
 

Doc/library/binascii.rst

@@ -50,11 +50,23 @@ The :mod:`binascii` module defines the following functions:
       Added the *backtick* parameter.
 
 
-.. function:: a2b_base64(string)
+.. function:: a2b_base64(string, strict_mode=False)
 
    Convert a block of base64 data back to binary and return the binary data. More
    than one line may be passed at a time.
 
+   If *strict_mode* is true, only valid base64 data will be converted. Invalid base64
+   data will raise :exc:`binascii.Error`.
+
+   Valid base64:
+      * Conforms to :rfc:`3548`.
+      * Contains only characters from the base64 alphabet.
+      * Contains no excess data after padding (including excess padding, newlines, etc.).
+      * Does not start with a padding.
+
+   .. versionchanged:: 3.11
+      Added the *strict_mode* parameter.
+
 
 .. function:: b2a_base64(data, *, newline=True)
 

Lib/pdb.py

@@ -80,9 +80,12 @@
 import signal
 import inspect
 import tokenize
+import functools
 import traceback
 import linecache
 
+from typing import Union
+
 
 class Restart(Exception):
     """Causes a debugger to be restarted for the debugged python program."""
@@ -128,6 +131,77 @@ def __repr__(self):
         return self
 
 
+class ScriptTarget(str):
+    def __new__(cls, val):
+        # Mutate self to be the "real path".
+        res = super().__new__(cls, os.path.realpath(val))
+
+        # Store the original path for error reporting.
+        res.orig = val
+
+        return res
+
+    def check(self):
+        if not os.path.exists(self):
+            print('Error:', self.orig, 'does not exist')
+            sys.exit(1)
+
+        # Replace pdb's dir with script's dir in front of module search path.
+        sys.path[0] = os.path.dirname(self)
+
+    @property
+    def filename(self):
+        return self
+
+    @property
+    def namespace(self):
+        return dict(
+            __name__='__main__',
+            __file__=self,
+            __builtins__=__builtins__,
+        )
+
+    @property
+    def code(self):
+        with io.open(self) as fp:
+            return f"exec(compile({fp.read()!r}, {self!r}, 'exec'))"
+
+
+class ModuleTarget(str):
+    def check(self):
+        pass
+
+    @functools.cached_property
+    def _details(self):
+        import runpy
+        return runpy._get_module_details(self)
+
+    @property
+    def filename(self):
+        return self.code.co_filename
+
+    @property
+    def code(self):
+        name, spec, code = self._details
+        return code
+
+    @property
+    def _spec(self):
+        name, spec, code = self._details
+        return spec
+
+    @property
+    def namespace(self):
+        return dict(
+            __name__='__main__',
+            __file__=os.path.normcase(os.path.abspath(self.filename)),
+            __package__=self._spec.parent,
+            __loader__=self._spec.loader,
+            __spec__=self._spec,
+            __builtins__=__builtins__,
+        )
+
+
 # Interaction prompt line will separate file and call info from code
 # text using value of line_prefix string.  A newline and arrow may
 # be to your liking.  You can set it once pdb is imported using the
@@ -1538,49 +1612,26 @@ def lookupmodule(self, filename):
                 return fullname
         return None
 
-    def _runmodule(self, module_name):
+    def _run(self, target: Union[ModuleTarget, ScriptTarget]):
+        # When bdb sets tracing, a number of call and line events happen
+        # BEFORE debugger even reaches user's code (and the exact sequence of
+        # events depends on python version). Take special measures to
+        # avoid stopping before reaching the main script (see user_line and
+        # user_call for details).
         self._wait_for_mainpyfile = True
         self._user_requested_quit = False
-        import runpy
-        mod_name, mod_spec, code = runpy._get_module_details(module_name)
-        self.mainpyfile = self.canonic(code.co_filename)
-        import __main__
-        __main__.__dict__.clear()
-        __main__.__dict__.update({
-            "__name__": "__main__",
-            "__file__": self.mainpyfile,
-            "__package__": mod_spec.parent,
-            "__loader__": mod_spec.loader,
-            "__spec__": mod_spec,
-            "__builtins__": __builtins__,
-        })
-        self.run(code)
-
-    def _runscript(self, filename):
-        # The script has to run in __main__ namespace (or imports from
-        # __main__ will break).
-        #
-        # So we clear up the __main__ and set several special variables
-        # (this gets rid of pdb's globals and cleans old variables on restarts).
+
+        self.mainpyfile = self.canonic(target.filename)
+
+        # The target has to run in __main__ namespace (or imports from
+        # __main__ will break). Clear __main__ and replace with
+        # the target namespace.
         import __main__
         __main__.__dict__.clear()
-        __main__.__dict__.update({"__name__"    : "__main__",
-                                  "__file__"    : filename,
-                                  "__builtins__": __builtins__,
-                                 })
+        __main__.__dict__.update(target.namespace)
+
+        self.run(target.code)
 
-        # When bdb sets tracing, a number of call and line events happens
-        # BEFORE debugger even reaches user's code (and the exact sequence of
-        # events depends on python version). So we take special measures to
-        # avoid stopping before we reach the main script (see user_line and
-        # user_call for details).
-        self._wait_for_mainpyfile = True
-        self.mainpyfile = self.canonic(filename)
-        self._user_requested_quit = False
-        with io.open_code(filename) as fp:
-            statement = "exec(compile(%r, %r, 'exec'))" % \
-                        (fp.read(), self.mainpyfile)
-        self.run(statement)
 
 # Collect all command help into docstring, if not run with -OO
 
@@ -1669,6 +1720,7 @@ def help():
 To let the script run up to a given line X in the debugged file, use
 "-c 'until X'"."""
 
+
 def main():
     import getopt
 
@@ -1678,28 +1730,19 @@ def main():
         print(_usage)
         sys.exit(2)
 
-    commands = []
-    run_as_module = False
-    for opt, optarg in opts:
-        if opt in ['-h', '--help']:
-            print(_usage)
-            sys.exit()
-        elif opt in ['-c', '--command']:
-            commands.append(optarg)
-        elif opt in ['-m']:
-            run_as_module = True
-
-    mainpyfile = args[0]     # Get script filename
-    if not run_as_module and not os.path.exists(mainpyfile):
-        print('Error:', mainpyfile, 'does not exist')
-        sys.exit(1)
+    if any(opt in ['-h', '--help'] for opt, optarg in opts):
+        print(_usage)
+        sys.exit()
 
-    sys.argv[:] = args      # Hide "pdb.py" and pdb options from argument list
+    commands = [optarg for opt, optarg in opts if opt in ['-c', '--command']]
 
-    if not run_as_module:
-        mainpyfile = os.path.realpath(mainpyfile)
-        # Replace pdb's dir with script's dir in front of module search path.
-        sys.path[0] = os.path.dirname(mainpyfile)
+    module_indicated = any(opt in ['-m'] for opt, optarg in opts)
+    cls = ModuleTarget if module_indicated else ScriptTarget
+    target = cls(args[0])
+
+    target.check()
+
+    sys.argv[:] = args      # Hide "pdb.py" and pdb options from argument list
 
     # Note on saving/restoring sys.argv: it's a good idea when sys.argv was
     # modified by the script being debugged. It's a bad idea when it was
@@ -1709,15 +1752,12 @@ def main():
     pdb.rcLines.extend(commands)
     while True:
         try:
-            if run_as_module:
-                pdb._runmodule(mainpyfile)
-            else:
-                pdb._runscript(mainpyfile)
+            pdb._run(target)
             if pdb._user_requested_quit:
                 break
             print("The program finished and will be restarted")
         except Restart:
-            print("Restarting", mainpyfile, "with arguments:")
+            print("Restarting", target, "with arguments:")
             print("\t" + " ".join(sys.argv[1:]))
         except SystemExit:
             # In most cases SystemExit does not warrant a post-mortem session.
@@ -1732,7 +1772,7 @@ def main():
             print("Running 'cont' or 'step' will restart the program")
             t = sys.exc_info()[2]
             pdb.interaction(None, t)
-            print("Post mortem debugger finished. The " + mainpyfile +
+            print("Post mortem debugger finished. The " + target +
                   " will be restarted")
 
 

Lib/test/test_binascii.py

@@ -114,6 +114,47 @@ def addnoise(line):
         # empty strings. TBD: shouldn't it raise an exception instead ?
         self.assertEqual(binascii.a2b_base64(self.type2test(fillers)), b'')
 
+    def test_base64_strict_mode(self):
+        # Test base64 with strict mode on
+        def _assertRegexTemplate(assert_regex: str, data: bytes, non_strict_mode_expected_result: bytes):
+            with self.assertRaisesRegex(binascii.Error, assert_regex):
+                binascii.a2b_base64(self.type2test(data), strict_mode=True)
+            self.assertEqual(binascii.a2b_base64(self.type2test(data), strict_mode=False),
+                             non_strict_mode_expected_result)
+            self.assertEqual(binascii.a2b_base64(self.type2test(data)),
+                             non_strict_mode_expected_result)
+
+        def assertExcessData(data, non_strict_mode_expected_result: bytes):
+            _assertRegexTemplate(r'(?i)Excess data', data, non_strict_mode_expected_result)
+
+        def assertNonBase64Data(data, non_strict_mode_expected_result: bytes):
+            _assertRegexTemplate(r'(?i)Only base64 data', data, non_strict_mode_expected_result)
+
+        def assertMalformedPadding(data, non_strict_mode_expected_result: bytes):
+            _assertRegexTemplate(r'(?i)Leading padding', data, non_strict_mode_expected_result)
+
+        # Test excess data exceptions
+        assertExcessData(b'ab==a', b'i')
+        assertExcessData(b'ab===', b'i')
+        assertExcessData(b'ab==:', b'i')
+        assertExcessData(b'abc=a', b'i\xb7')
+        assertExcessData(b'abc=:', b'i\xb7')
+        assertExcessData(b'ab==\n', b'i')
+
+        # Test non-base64 data exceptions
+        assertNonBase64Data(b'\nab==', b'i')
+        assertNonBase64Data(b'ab:(){:|:&};:==', b'i')
+        assertNonBase64Data(b'a\nb==', b'i')
+        assertNonBase64Data(b'a\x00b==', b'i')
+
+        # Test malformed padding
+        assertMalformedPadding(b'=', b'')
+        assertMalformedPadding(b'==', b'')
+        assertMalformedPadding(b'===', b'')
+        assertMalformedPadding(b'ab=c=', b'i\xb7')
+        assertMalformedPadding(b'ab=ab==', b'i\xb6\x9b')
+
+
     def test_base64errors(self):
         # Test base64 with invalid padding
         def assertIncorrectPadding(data):

Lib/test/test_pdb.py

@@ -362,7 +362,7 @@ def test_pdb_breakpoints_preserved_across_interactive_sessions():
     1   breakpoint   keep yes   at ...test_pdb.py:...
     2   breakpoint   keep yes   at ...test_pdb.py:...
     (Pdb) break pdb.find_function
-    Breakpoint 3 at ...pdb.py:94
+    Breakpoint 3 at ...pdb.py:97
     (Pdb) break
     Num Type         Disp Enb   Where
     1   breakpoint   keep yes   at ...test_pdb.py:...

Lib/test/test_pyclbr.py

@@ -222,7 +222,11 @@ def test_others(self):
         cm('pickle', ignore=('partial', 'PickleBuffer'))
         cm('aifc', ignore=('_aifc_params',))  # set with = in module
         cm('sre_parse', ignore=('dump', 'groups', 'pos')) # from sre_constants import *; property
-        cm('pdb')
+        cm(
+            'pdb',
+            # pyclbr does not handle elegantly `typing` or properties
+            ignore=('Union', 'ModuleTarget', 'ScriptTarget'),
+        )
         cm('pydoc', ignore=('input', 'output',)) # properties
 
         # Tests for modules inside packages

Misc/NEWS.d/next/Documentation/2021-07-18-22-43-14.bpo-44561.T7HpWm.rst

@@ -0,0 +1,3 @@
+Update of three expired hyperlinks in Doc/distributing/index.rst:
+"Project structure", "Building and packaging the project", and "Uploading the
+project to the Python Packaging Index".

Misc/NEWS.d/next/Library/2021-01-31-18-24-54.bpo-43086.2_P-SH.rst

@@ -0,0 +1,3 @@
+Added a new optional :code:`strict_mode` parameter to *binascii.a2b_base64*.
+When :code:`scrict_mode` is set to :code:`True`, the *a2b_base64* function will accept only valid base64 content.
+More details about what "valid base64 content" is, can be found in the function's documentation.

Misc/NEWS.d/next/Library/2021-07-02-18-17-56.bpo-44554.aBUmJo.rst

@@ -0,0 +1 @@
+Refactor argument processing in :func:pdb.main to simplify detection of errors in input loading and clarify behavior around module or script invocation.