Merge with branch 3.3 just before the conversion to git.

Author: Anselm Kruis

Doc/README.txt

@@ -108,27 +108,3 @@ and we will process your request as soon as possible.
 
 If you want to help the Documentation Team, you are always welcome.  Just send
 a mail to [email protected].
-
-
-Copyright notice
-================
-
-The Python source is copyrighted, but you can freely use and copy it
-as long as you don't change or remove the copyright notice:
-
-----------------------------------------------------------------------
-Copyright (c) 2000-2014 Python Software Foundation.
-All rights reserved.
-
-Copyright (c) 2000 BeOpen.com.
-All rights reserved.
-
-Copyright (c) 1995-2000 Corporation for National Research Initiatives.
-All rights reserved.
-
-Copyright (c) 1991-1995 Stichting Mathematisch Centrum.
-All rights reserved.
-
-See the file "license.rst" for information on usage and redistribution
-of this file, and for a DISCLAIMER OF ALL WARRANTIES.
-----------------------------------------------------------------------

Doc/copyright.rst

@@ -4,7 +4,7 @@ Copyright
 
 Python and this documentation is:
 
-Copyright © 2001-2015 Python Software Foundation. All rights reserved.
+Copyright © 2001-2016 Python Software Foundation. All rights reserved.
 
 Copyright © 2000 BeOpen.com. All rights reserved.
 

Doc/howto/urllib2.rst

@@ -538,6 +538,11 @@ setting up a `Basic Authentication`_ handler: ::
     through a proxy.  However, this can be enabled by extending urllib.request as
     shown in the recipe [#]_.
 
+.. note::
+
+   `HTTP_PROXY`` will be ignored if a variable ``REQUEST_METHOD`` is set; see
+   the documentation on :func:`~urllib.request.getproxies`.
+
 
 Sockets and Layers
 ==================

Doc/library/urllib.request.rst

@@ -161,6 +161,14 @@ The :mod:`urllib.request` module defines the following functions:
    cannot find it, looks for proxy information from Mac OSX System
    Configuration for Mac OS X and Windows Systems Registry for Windows.
 
+   .. note::
+
+      If the environment variable ``REQUEST_METHOD`` is set, which usually
+      indicates your script is running in a CGI environment, the environment
+      variable ``HTTP_PROXY`` (uppercase ``_PROXY``) will be ignored. This is
+      because that variable can be injected by a client using the "Proxy:" HTTP
+      header. If you need to use an HTTP proxy in a CGI environment use
+      ``ProxyHandler`` explicitly.
 
 The following classes are provided:
 
@@ -265,6 +273,11 @@ The following classes are provided:
 
    To disable autodetected proxy pass an empty dictionary.
 
+   .. note::
+
+      ``HTTP_PROXY`` will be ignored if a variable ``REQUEST_METHOD`` is set;
+      see the documentation on :func:`~urllib.request.getproxies`.
+
 
 .. class:: HTTPPasswordMgr()
 

Doc/license.rst

@@ -84,7 +84,7 @@ Terms and conditions for accessing or otherwise using Python
    analyze, test, perform and/or display publicly, prepare derivative works,
    distribute, and otherwise use Python |release| alone or in any derivative
    version, provided, however, that PSF's License Agreement and PSF's notice of
-   copyright, i.e., "Copyright © 2001-2015 Python Software Foundation; All Rights
+   copyright, i.e., "Copyright © 2001-2017 Python Software Foundation; All Rights
    Reserved" are retained in Python |release| alone or in any derivative version
    prepared by Licensee.
 

LICENSE

@@ -74,8 +74,9 @@ analyze, test, perform and/or display publicly, prepare derivative works,
 distribute, and otherwise use Python alone or in any derivative version,
 provided, however, that PSF's License Agreement and PSF's notice of copyright,
 i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
-2011, 2012, 2013, 2014, 2015 Python Software Foundation; All Rights Reserved"
-are retained in Python alone or in any derivative version prepared by Licensee.
+2011, 2012, 2013, 2014, 2015, 2016, 2017 Python Software Foundation; All Rights
+Reserved" are retained in Python alone or in any derivative version prepared by
+Licensee.
 
 3. In the event Licensee prepares a derivative work that is based on
 or incorporates Python or any part thereof, and wants to make

Lib/ctypes/test/test_callbacks.py

@@ -1,3 +1,4 @@
+import functools
 import unittest
 from ctypes import *
 import _ctypes_test
@@ -243,6 +244,40 @@ def callback(a, b, c, d, e):
         self.assertEqual(result,
                          callback(1.1*1.1, 2.2*2.2, 3.3*3.3, 4.4*4.4, 5.5*5.5))
 
+    def test_callback_large_struct(self):
+        class Check: pass
+
+        class X(Structure):
+            _fields_ = [
+                ('first', c_ulong),
+                ('second', c_ulong),
+                ('third', c_ulong),
+            ]
+
+        def callback(check, s):
+            check.first = s.first
+            check.second = s.second
+            check.third = s.third
+
+        check = Check()
+        s = X()
+        s.first = 0xdeadbeef
+        s.second = 0xcafebabe
+        s.third = 0x0bad1dea
+
+        CALLBACK = CFUNCTYPE(None, X)
+        dll = CDLL(_ctypes_test.__file__)
+        func = dll._testfunc_cbk_large_struct
+        func.argtypes = (X, CALLBACK)
+        func.restype = None
+        # the function just calls the callback with the passed structure
+        func(s, CALLBACK(functools.partial(callback, check)))
+        self.assertEqual(check.first, s.first)
+        self.assertEqual(check.second, s.second)
+        self.assertEqual(check.third, s.third)
+        self.assertEqual(check.first, 0xdeadbeef)
+        self.assertEqual(check.second, 0xcafebabe)
+        self.assertEqual(check.third, 0x0bad1dea)
 
 ################################################################
 

Lib/distutils/__init__.py

@@ -8,10 +8,6 @@
    setup (...)
 """
 
-# Distutils version
-#
-# Updated automatically by the Python release process.
-#
-#--start constants--
-__version__ = "3.3.6"
-#--end constants--
+import sys
+
+__version__ = sys.version[:sys.version.index(' ')]

Lib/distutils/config.py

@@ -21,7 +21,7 @@
 class PyPIRCCommand(Command):
     """Base command that knows how to handle the .pypirc file
     """
-    DEFAULT_REPOSITORY = 'https://pypi.python.org/pypi'
+    DEFAULT_REPOSITORY = 'https://upload.pypi.org/legacy/'
     DEFAULT_REALM = 'pypi'
     repository = None
     realm = None

Lib/distutils/tests/test_config.py

@@ -87,7 +87,7 @@ def test_server_registration(self):
 
         config = list(sorted(config.items()))
         waited = [('password', 'secret'), ('realm', 'pypi'),
-                  ('repository', 'https://pypi.python.org/pypi'),
+                  ('repository', 'https://upload.pypi.org/legacy/'),
                   ('server', 'server1'), ('username', 'me')]
         self.assertEqual(config, waited)
 
@@ -96,7 +96,7 @@ def test_server_registration(self):
         config = cmd._read_pypirc()
         config = list(sorted(config.items()))
         waited = [('password', 'secret'), ('realm', 'pypi'),
-                  ('repository', 'https://pypi.python.org/pypi'),
+                  ('repository', 'https://upload.pypi.org/legacy/'),
                   ('server', 'server-login'), ('username', 'tarek')]
         self.assertEqual(config, waited)
 

Lib/distutils/tests/test_upload.py

@@ -127,7 +127,7 @@ def test_upload(self):
         self.assertTrue(headers['Content-type'].startswith('multipart/form-data'))
         self.assertEqual(self.last_open.req.get_method(), 'POST')
         self.assertEqual(self.last_open.req.get_full_url(),
-                         'https://pypi.python.org/pypi')
+                         'https://upload.pypi.org/legacy/')
         self.assertIn(b'xxx', self.last_open.req.data)
 
         # The PyPI response body was echoed

Lib/gettext.py

@@ -57,55 +57,147 @@
 
 _default_localedir = os.path.join(sys.base_prefix, 'share', 'locale')
 
+# Expression parsing for plural form selection.
+#
+# The gettext library supports a small subset of C syntax.  The only
+# incompatible difference is that integer literals starting with zero are
+# decimal.
+#
+# https://www.gnu.org/software/gettext/manual/gettext.html#Plural-forms
+# http://git.savannah.gnu.org/cgit/gettext.git/tree/gettext-runtime/intl/plural.y
+
+_token_pattern = re.compile(r"""
+        (?P<WHITESPACES>[ \t]+)                    | # spaces and horizontal tabs
+        (?P<NUMBER>[0-9]+\b)                       | # decimal integer
+        (?P<NAME>n\b)                              | # only n is allowed
+        (?P<PARENTHESIS>[()])                      |
+        (?P<OPERATOR>[-*/%+?:]|[><!]=?|==|&&|\|\|) | # !, *, /, %, +, -, <, >,
+                                                     # <=, >=, ==, !=, &&, ||,
+                                                     # ? :
+                                                     # unary and bitwise ops
+                                                     # not allowed
+        (?P<INVALID>\w+|.)                           # invalid token
+    """, re.VERBOSE|re.DOTALL)
+
+def _tokenize(plural):
+    for mo in re.finditer(_token_pattern, plural):
+        kind = mo.lastgroup
+        if kind == 'WHITESPACES':
+            continue
+        value = mo.group(kind)
+        if kind == 'INVALID':
+            raise ValueError('invalid token in plural form: %s' % value)
+        yield value
+    yield ''
+
+def _error(value):
+    if value:
+        return ValueError('unexpected token in plural form: %s' % value)
+    else:
+        return ValueError('unexpected end of plural form')
+
+_binary_ops = (
+    ('||',),
+    ('&&',),
+    ('==', '!='),
+    ('<', '>', '<=', '>='),
+    ('+', '-'),
+    ('*', '/', '%'),
+)
+_binary_ops = {op: i for i, ops in enumerate(_binary_ops, 1) for op in ops}
+_c2py_ops = {'||': 'or', '&&': 'and', '/': '//'}
+
+def _parse(tokens, priority=-1):
+    result = ''
+    nexttok = next(tokens)
+    while nexttok == '!':
+        result += 'not '
+        nexttok = next(tokens)
+
+    if nexttok == '(':
+        sub, nexttok = _parse(tokens)
+        result = '%s(%s)' % (result, sub)
+        if nexttok != ')':
+            raise ValueError('unbalanced parenthesis in plural form')
+    elif nexttok == 'n':
+        result = '%s%s' % (result, nexttok)
+    else:
+        try:
+            value = int(nexttok, 10)
+        except ValueError:
+            raise _error(nexttok) from None
+        result = '%s%d' % (result, value)
+    nexttok = next(tokens)
+
+    j = 100
+    while nexttok in _binary_ops:
+        i = _binary_ops[nexttok]
+        if i < priority:
+            break
+        # Break chained comparisons
+        if i in (3, 4) and j in (3, 4):  # '==', '!=', '<', '>', '<=', '>='
+            result = '(%s)' % result
+        # Replace some C operators by their Python equivalents
+        op = _c2py_ops.get(nexttok, nexttok)
+        right, nexttok = _parse(tokens, i + 1)
+        result = '%s %s %s' % (result, op, right)
+        j = i
+    if j == priority == 4:  # '<', '>', '<=', '>='
+        result = '(%s)' % result
+
+    if nexttok == '?' and priority <= 0:
+        if_true, nexttok = _parse(tokens, 0)
+        if nexttok != ':':
+            raise _error(nexttok)
+        if_false, nexttok = _parse(tokens)
+        result = '%s if %s else %s' % (if_true, result, if_false)
+        if priority == 0:
+            result = '(%s)' % result
+
+    return result, nexttok
+
+def _as_int(n):
+    try:
+        i = round(n)
+    except TypeError:
+        raise TypeError('Plural value must be an integer, got %s' %
+                        (n.__class__.__name__,)) from None
+    return n
 
 def c2py(plural):
     """Gets a C expression as used in PO files for plural forms and returns a
-    Python lambda function that implements an equivalent expression.
+    Python function that implements an equivalent expression.
     """
-    # Security check, allow only the "n" identifier
-    import token, tokenize
-    tokens = tokenize.generate_tokens(io.StringIO(plural).readline)
-    try:
-        danger = [x for x in tokens if x[0] == token.NAME and x[1] != 'n']
-    except tokenize.TokenError:
-        raise ValueError('plural forms expression error, maybe unbalanced parenthesis')
-    else:
-        if danger:
-            raise ValueError('plural forms expression could be dangerous')
-
-    # Replace some C operators by their Python equivalents
-    plural = plural.replace('&&', ' and ')
-    plural = plural.replace('||', ' or ')
-
-    expr = re.compile(r'\!([^=])')
-    plural = expr.sub(' not \\1', plural)
-
-    # Regular expression and replacement function used to transform
-    # "a?b:c" to "b if a else c".
-    expr = re.compile(r'(.*?)\?(.*?):(.*)')
-    def repl(x):
-        return "(%s if %s else %s)" % (x.group(2), x.group(1),
-                                       expr.sub(repl, x.group(3)))
-
-    # Code to transform the plural expression, taking care of parentheses
-    stack = ['']
-    for c in plural:
-        if c == '(':
-            stack.append('')
-        elif c == ')':
-            if len(stack) == 1:
-                # Actually, we never reach this code, because unbalanced
-                # parentheses get caught in the security check at the
-                # beginning.
-                raise ValueError('unbalanced parenthesis in plural form')
-            s = expr.sub(repl, stack.pop())
-            stack[-1] += '(%s)' % s
-        else:
-            stack[-1] += c
-    plural = expr.sub(repl, stack.pop())
-
-    return eval('lambda n: int(%s)' % plural)
 
+    if len(plural) > 1000:
+        raise ValueError('plural form expression is too long')
+    try:
+        result, nexttok = _parse(_tokenize(plural))
+        if nexttok:
+            raise _error(nexttok)
+
+        depth = 0
+        for c in result:
+            if c == '(':
+                depth += 1
+                if depth > 20:
+                    # Python compiler limit is about 90.
+                    # The most complex example has 2.
+                    raise ValueError('plural form expression is too complex')
+            elif c == ')':
+                depth -= 1
+
+        ns = {'_as_int': _as_int}
+        exec('''if True:
+            def func(n):
+                if not isinstance(n, int):
+                    n = _as_int(n)
+                return int(%s)
+            ''' % result, ns)
+        return ns['func']
+    except RuntimeError:
+        # Recursion error can be raised in _parse() or exec().
+        raise ValueError('plural form expression is too complex')
 
 
 def _expand_lang(loc):