Merge branch 'master' of https://github.com/python/cpython into asyncio

* 'master' of https://github.com/python/cpython: (32 commits)
  Conceptually, roots is a set.  Also searching it as a set is a tiny bit faster (python#3338)
  bpo-31343: Include sys/sysmacros.h (python#3318)
  bpo-30102: Call OPENSSL_add_all_algorithms_noconf (python#3112)
  Prevent a few make suspicious warnings. (python#3341)
  Include additional changes to support blurbified NEWS (python#3340)
  Simplify NEWS entry to prevent suspicious warnings. (python#3339)
  bpo-31347: _PyObject_FastCall_Prepend: do not call memcpy if args might not be null (python#3329)
  Revert "bpo-17852: Maintain a list of BufferedWriter objects.  Flush them on exit. (python#1908)" (python#3337)
  bpo-17852: Maintain a list of BufferedWriter objects.  Flush them on exit. (python#1908)
  Fix terminology in comment and add more design rationale. (python#3335)
  Add comment to explain the implications of not sorting keywords (python#3331)
  bpo-31170: Update libexpat from 2.2.3 to 2.2.4 (python#3315)
  bpo-28411: Remove "modules" field from Py_InterpreterState. (python#1638)
  random_triangular:  sqrt() is more accurate than **0.5 (python#3317)
  Travis: use ccache (python#3307)
  remove IRIX support (closes bpo-31341) (python#3310)
  Code clean-up.  Remove unnecessary pre-increment before the loop starts. (python#3312)
  Regen Moduls/clinic/_ssl.c.h (pythonGH-3320)
  bpo-30502: Fix handling of long oids in ssl. (python#2909)
  Cache externals, depending on changes to PCbuild (python#3308)
  ...

Author: Jimmy Lai

.github/CODEOWNERS

@@ -24,11 +24,11 @@
 Python/bootstrap_hash.c       @python/crypto-team
 
 # Email and related
-**/*mail*                     @bitdancer
-**/*smtp*                     @bitdancer
-**/*mime*                     @bitdancer
-**/*imap*                     @bitdancer
-**/*poplib*                   @bitdancer
+**/*mail*                     @python/email-team
+**/*smtp*                     @python/email-team
+**/*mime*                     @python/email-team
+**/*imap*                     @python/email-team
+**/*poplib*                   @python/email-team
 
 # subprocess
 **/*subprocess*               @gpshead
@@ -40,3 +40,12 @@ Python/bootstrap_hash.c       @python/crypto-team
 # Windows installer packages
 /Tools/msi/                   @python/windows-team
 /Tools/nuget/                 @python/windows-team
+
+**/*itertools*                @rhettinger
+**/*collections*              @rhettinger
+**/*random*                   @rhettinger
+**/*queue*                    @rhettinger
+**/*bisect*                   @rhettinger
+**/*heapq*                    @rhettinger
+**/*functools*                @ncoghlan @rhettinger
+**/*decimal*                  @rhettinger @skrah

.github/appveyor.yml

@@ -5,6 +5,8 @@ branches:
     - master
     - /\d\.\d/
     - buildbot-custom
+cache:
+  - externals -> PCbuild\*
 build_script:
 - cmd: PCbuild\build.bat -e
 - cmd: PCbuild\win32\python.exe -m test.pythoninfo

.travis.yml

@@ -3,8 +3,10 @@ dist: trusty
 sudo: false
 group: beta
 
-# To cache doc-building dependencies.
-cache: pip
+# To cache doc-building dependencies and C compiler output.
+cache:
+    - pip
+    - ccache
 
 branches:
   only:
@@ -41,7 +43,7 @@ matrix:
         - cd Doc
         # Sphinx is pinned so that new versions that introduce new warnings won't suddenly cause build failures.
         # (Updating the version is fine as long as no warnings are raised by doing so.)
-        - python -m pip install sphinx~=1.6.1
+        - python -m pip install sphinx~=1.6.1 blurb
       script:
         - make check suspicious html SPHINXOPTS="-q -W -j4"
     - os: linux

Doc/Makefile

@@ -6,6 +6,7 @@
 # You can set these variables from the command line.
 PYTHON       = python3
 SPHINXBUILD  = sphinx-build
+BLURB = $(PYTHON) -m blurb
 PAPER        =
 SOURCES      =
 DISTVERSION  = $(shell $(PYTHON) tools/extensions/patchlevel.py)
@@ -38,6 +39,20 @@ help:
 	@echo "  serve      to serve the documentation on the localhost (8000)"
 
 build:
+	-mkdir -p build
+# Look first for a Misc/NEWS file (building from a source release tarball
+# or old repo) and use that, otherwise look for a Misc/NEWS.d directory
+# (building from a newer repo) and use blurb to generate the NEWS file.
+	@if [ -f  ../Misc/NEWS ] ; then \
+		echo "Using existing Misc/NEWS file"; \
+		cp ../Misc/NEWS build/NEWS; \
+	elif [ -d ../Misc/NEWS.d ]; then \
+		echo "Building NEWS from Misc/NEWS.d with blurb"; \
+		$(BLURB) merge -f build/NEWS; \
+	else \
+		echo "Neither Misc/NEWS.d nor Misc/NEWS found; cannot build docs"; \
+		exit 1; \
+	fi
 	$(SPHINXBUILD) $(ALLSPHINXOPTS)
 	@echo
 
@@ -107,7 +122,7 @@ clean:
 
 venv:
 	$(PYTHON) -m venv venv
-	./venv/bin/python3 -m pip install -U Sphinx
+	./venv/bin/python3 -m pip install -U Sphinx blurb
 
 dist:
 	rm -rf dist

Doc/c-api/import.rst

@@ -204,6 +204,13 @@ Importing Modules
    Return the dictionary used for the module administration (a.k.a.
    ``sys.modules``).  Note that this is a per-interpreter variable.
 
+.. c:function:: PyObject* PyImport_GetModule(PyObject *name)
+
+   Return the already imported module with the given name.  If the
+   module has not been imported yet then returns NULL but does not set
+   an error.  Returns NULL and sets an error if the lookup failed.
+
+   .. versionadded:: 3.7
 
 .. c:function:: PyObject* PyImport_GetImporter(PyObject *path)
 

Doc/distutils/apiref.rst

@@ -1086,19 +1086,16 @@ other utility module.
 
    Return a string that identifies the current platform.  This is used mainly to
    distinguish platform-specific build directories and platform-specific built
-   distributions.  Typically includes the OS name and version and the architecture
-   (as supplied by 'os.uname()'), although the exact information included depends
-   on the OS; eg. for IRIX the architecture isn't particularly important (IRIX only
-   runs on SGI hardware), but for Linux the kernel version isn't particularly
-   important.
+   distributions.  Typically includes the OS name and version and the
+   architecture (as supplied by 'os.uname()'), although the exact information
+   included depends on the OS; e.g., on Linux, the kernel version isn't
+   particularly important.
 
    Examples of returned values:
 
    * ``linux-i586``
    * ``linux-alpha``
    * ``solaris-2.6-sun4u``
-   * ``irix-5.3``
-   * ``irix64-6.2``
 
    For non-POSIX platforms, currently just returns ``sys.platform``.
 

Doc/library/aifc.rst

@@ -18,12 +18,6 @@ AIFF is Audio Interchange File Format, a format for storing digital audio
 samples in a file.  AIFF-C is a newer version of the format that includes the
 ability to compress the audio data.
 
-.. note::
-
-   Some operations may only work under IRIX; these will raise :exc:`ImportError`
-   when attempting to import the :mod:`cl` module, which is only available on
-   IRIX.
-
 Audio files have a number of parameters that describe the audio data. The
 sampling rate or frame rate is the number of times per second the sound is
 sampled.  The number of channels indicate if the audio is mono, stereo, or

Doc/library/argparse.rst

@@ -898,6 +898,8 @@ values are:
      usage: PROG [-h] foo [foo ...]
      PROG: error: too few arguments
 
+.. _`argparse.REMAINDER`:
+
 * ``argparse.REMAINDER``.  All the remaining command-line arguments are gathered
   into a list.  This is commonly useful for command line utilities that dispatch
   to other command line utilities::
@@ -1324,8 +1326,11 @@ The parse_args() method
    created and how they are assigned. See the documentation for
    :meth:`add_argument` for details.
 
-   By default, the argument strings are taken from :data:`sys.argv`, and a new empty
-   :class:`Namespace` object is created for the attributes.
+   * args_ - List of strings to parse.  The default is taken from
+     :data:`sys.argv`.
+
+   * namespace_ - An object to take the attributes.  The default is a new empty
+     :class:`Namespace` object.
 
 
 Option value syntax
@@ -1467,6 +1472,7 @@ unambiguous (the prefix matches a unique option)::
 An error is produced for arguments that could produce more than one options.
 This feature can be disabled by setting :ref:`allow_abbrev` to ``False``.
 
+.. _args:
 
 Beyond ``sys.argv``
 ^^^^^^^^^^^^^^^^^^^
@@ -1488,6 +1494,7 @@ interactive prompt::
    >>> parser.parse_args(['1', '2', '3', '4', '--sum'])
    Namespace(accumulate=<built-in function sum>, integers=[1, 2, 3, 4])
 
+.. _namespace:
 
 The Namespace object
 ^^^^^^^^^^^^^^^^^^^^
@@ -2008,7 +2015,12 @@ A partial upgrade path from :mod:`optparse` to :mod:`argparse`:
 * Replace ``(options, args) = parser.parse_args()`` with ``args =
   parser.parse_args()`` and add additional :meth:`ArgumentParser.add_argument`
   calls for the positional arguments. Keep in mind that what was previously
-  called ``options``, now in :mod:`argparse` context is called ``args``.
+  called ``options``, now in the :mod:`argparse` context is called ``args``.
+
+* Replace :meth:`optparse.OptionParser.disable_interspersed_args`
+  by setting ``nargs`` of a positional argument to `argparse.REMAINDER`_, or
+  use :meth:`~ArgumentParser.parse_known_args` to collect unparsed argument
+  strings in a separate list.
 
 * Replace callback actions and the ``callback_*`` keyword arguments with
   ``type`` or ``action`` arguments.

Doc/library/string.rst

@@ -754,9 +754,21 @@ attributes:
   be set in the subclass's class namespace).
 
 * *idpattern* -- This is the regular expression describing the pattern for
-  non-braced placeholders (the braces will be added automatically as
-  appropriate).  The default value is the regular expression
-  ``[_a-z][_a-z0-9]*``.
+  non-braced placeholders.  The default value is the regular expression
+  ``[_a-z][_a-z0-9]*``.  If this is given and *braceidpattern* is ``None``
+  this pattern will also apply to braced placeholders.
+
+  .. versionchanged:: 3.7
+     *braceidpattern* can be used to define separate patterns used inside and
+     outside the braces.
+
+* *braceidpattern* -- This is like *idpattern* but describes the pattern for
+  braced placeholders.  Defaults to ``None`` which means to fall back to
+  *idpattern* (i.e. the same pattern is used both inside and outside braces).
+  If given, this allows you to define different patterns for braced and
+  unbraced placeholders.
+
+  .. versionadded:: 3.7
 
 * *flags* -- The regular expression flags that will be applied when compiling
   the regular expression used for recognizing substitutions.  The default value

Doc/library/sysconfig.rst

@@ -173,18 +173,15 @@ Other functions
 
    This is used mainly to distinguish platform-specific build directories and
    platform-specific built distributions.  Typically includes the OS name and
-   version and the architecture (as supplied by :func:`os.uname`), although the
-   exact information included depends on the OS; e.g. for IRIX the architecture
-   isn't particularly important (IRIX only runs on SGI hardware), but for Linux
-   the kernel version isn't particularly important.
+   version and the architecture (as supplied by 'os.uname()'), although the
+   exact information included depends on the OS; e.g., on Linux, the kernel
+   version isn't particularly important.
 
    Examples of returned values:
 
    - linux-i586
    - linux-alpha (?)
    - solaris-2.6-sun4u
-   - irix-5.3
-   - irix64-6.2
 
    Windows will return one of:
 

Doc/make.bat

@@ -6,18 +6,18 @@ pushd %~dp0
 set this=%~n0
 
 call ..\PCBuild\find_python.bat %PYTHON%
-if "%SPHINXBUILD%" EQU "" if "%PYTHON%" NEQ "" (
-    set SPHINXBUILD=%PYTHON%\..\Scripts\sphinx-build.exe
-    rem Cannot use %SPHINXBUILD% in the same block where we set it
-    if not exist "%PYTHON%\..\Scripts\sphinx-build.exe" (
+if not defined SPHINXBUILD if defined PYTHON (
+    %PYTHON% -c "import sphinx" > nul 2> nul
+    if errorlevel 1 (
         echo Installing sphinx with %PYTHON%
-        "%PYTHON%" -m pip install sphinx
+        %PYTHON% -m pip install sphinx
         if errorlevel 1 exit /B
     )
+    set SPHINXBUILD=%PYTHON% -c "import sphinx, sys; sys.argv[0] = 'sphinx-build'; sphinx.main()"
 )
 
-if "%PYTHON%" EQU "" set PYTHON=py
-if "%SPHINXBUILD%" EQU "" set SPHINXBUILD=sphinx-build
+if not defined PYTHON set PYTHON=py
+if not defined SPHINXBUILD set SPHINXBUILD=sphinx-build
 
 if "%1" NEQ "htmlhelp" goto :skiphhcsearch
 if exist "%HTMLHELP%" goto :skiphhcsearch
@@ -99,7 +99,7 @@ goto end
 if NOT "%PAPER%" == "" (
     set SPHINXOPTS=-D latex_elements.papersize=%PAPER% %SPHINXOPTS%
 )
-cmd /C %SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . %BUILDDIR%\%*
+cmd /C "%SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . %BUILDDIR%\%*"
 
 if "%1" EQU "htmlhelp" (
     cmd /C "%HTMLHELP%" build\htmlhelp\python%DISTVERSION:.=%.hhp

Doc/tools/susp-ignored.csv

@@ -325,9 +325,8 @@ whatsnew/3.5,,:warning,'WARNING:root:warning\n'
 whatsnew/3.5,,::,>>> addr6 = ipaddress.IPv6Address('::1')
 whatsnew/3.5,,:root,ERROR:root:exception
 whatsnew/3.5,,:exception,ERROR:root:exception
-whatsnew/changelog,,:version,import sys; I = version[:version.index(' ')]
+whatsnew/changelog,,`,'`'
 whatsnew/changelog,,:end,str[start:end]
 library/binascii,,`,'`'
 library/uu,,`,'`'
 whatsnew/3.7,,`,'`'
-whatsnew/changelog,,`,'`'

Doc/whatsnew/3.7.rst

@@ -223,6 +223,13 @@ New function :func:`os.register_at_fork` allows registering Python callbacks
 to be executed on a process fork.  (Contributed by Antoine Pitrou in
 :issue:`16500`.)
 
+string
+------
+
+:class:`string.Template` now lets you to optionally modify the regular
+expression pattern for braced placeholders and non-braced placeholders
+separately.  (Contributed by Barry Warsaw in :issue:`1198569`.)
+
 unittest.mock
 -------------
 
@@ -449,6 +456,9 @@ Changes in the Python API
   and module are affected by this change. (Contributed by INADA Naoki and
   Eugene Toder in :issue:`29463`.)
 
+* ``PyInterpreterState`` no longer has a ``modules`` field.  Instead use
+  ``sys.modules``.
+
 * The *mode* argument of :func:`os.makedirs` no longer affects the file
   permission bits of newly-created intermediate-level directories.
   To set their file permission bits you can set the umask before invoking

Doc/whatsnew/changelog.rst

@@ -4,4 +4,4 @@
 Changelog
 +++++++++
 
-.. miscnews:: ../../Misc/NEWS
+.. miscnews:: ../build/NEWS

Include/import.h

@@ -38,11 +38,25 @@ PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleObject(
     );
 #endif
 PyAPI_FUNC(PyObject *) PyImport_GetModuleDict(void);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(int) _PyImport_IsInitialized(PyInterpreterState *);
+#endif
+PyAPI_FUNC(PyObject *) PyImport_GetModule(PyObject *name);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(PyObject *) _PyImport_GetModule(PyObject *name);
+PyAPI_FUNC(PyObject *) _PyImport_GetModuleWithError(PyObject *name);
+PyAPI_FUNC(PyObject *) _PyImport_GetModuleId(struct _Py_Identifier *name);
+PyAPI_FUNC(int) _PyImport_SetModule(PyObject *name, PyObject *module);
+PyAPI_FUNC(int) _PyImport_SetModuleString(const char *name, PyObject* module);
+#endif
 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000
 PyAPI_FUNC(PyObject *) PyImport_AddModuleObject(
     PyObject *name
     );
 #endif
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(PyObject *) _PyImport_AddModuleObject(PyObject *, PyObject *);
+#endif
 PyAPI_FUNC(PyObject *) PyImport_AddModule(
     const char *name            /* UTF-8 encoded string */
     );
@@ -97,14 +111,19 @@ PyAPI_FUNC(int) _PyImport_ReleaseLock(void);
 PyAPI_FUNC(void) _PyImport_ReInitLock(void);
 
 PyAPI_FUNC(PyObject *) _PyImport_FindBuiltin(
-    const char *name            /* UTF-8 encoded string */
+    const char *name,            /* UTF-8 encoded string */
+    PyObject *modules
     );
 PyAPI_FUNC(PyObject *) _PyImport_FindExtensionObject(PyObject *, PyObject *);
+PyAPI_FUNC(PyObject *) _PyImport_FindExtensionObjectEx(PyObject *, PyObject *,
+                                                       PyObject *);
 PyAPI_FUNC(int) _PyImport_FixupBuiltin(
     PyObject *mod,
-    const char *name            /* UTF-8 encoded string */
+    const char *name,            /* UTF-8 encoded string */
+    PyObject *modules
     );
-PyAPI_FUNC(int) _PyImport_FixupExtensionObject(PyObject*, PyObject *, PyObject *);
+PyAPI_FUNC(int) _PyImport_FixupExtensionObject(PyObject*, PyObject *,
+                                               PyObject *, PyObject *);
 
 struct _inittab {
     const char *name;           /* ASCII encoded string */

Include/modsupport.h

@@ -191,6 +191,10 @@ PyAPI_FUNC(int) PyModule_ExecDef(PyObject *module, PyModuleDef *def);
 
 PyAPI_FUNC(PyObject *) PyModule_Create2(struct PyModuleDef*,
                                      int apiver);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(PyObject *) _PyModule_CreateInitialized(struct PyModuleDef*,
+                                                   int apiver);
+#endif
 
 #ifdef Py_LIMITED_API
 #define PyModule_Create(module) \

Include/pystate.h

@@ -52,7 +52,6 @@ typedef struct _is {
 
     int64_t id;
 
-    PyObject *modules;
     PyObject *modules_by_index;
     PyObject *sysdict;
     PyObject *builtins;