bpo-23403: Bump pickle.DEFAULT_PROTOCOL to 4 (#6355)

This makes performance better and produces shorter pickles. This change is backwards compatible up to the oldest currently supported version of Python (3.4).

Author: ambv

Doc/library/pickle.rst

@@ -135,14 +135,14 @@ to read the pickle produced.
   information about improvements brought by protocol 2.
 
 * Protocol version 3 was added in Python 3.0.  It has explicit support for
-  :class:`bytes` objects and cannot be unpickled by Python 2.x.  This is
-  the default protocol, and the recommended protocol when compatibility with
-  other Python 3 versions is required.
+  :class:`bytes` objects and cannot be unpickled by Python 2.x.  This was
+  the default protocol in Python 3.0--3.7.
 
 * Protocol version 4 was added in Python 3.4.  It adds support for very large
   objects, pickling more kinds of objects, and some data format
-  optimizations.  Refer to :pep:`3154` for information about improvements
-  brought by protocol 4.
+  optimizations.  It is the default protocol starting with Python 3.8.
+  Refer to :pep:`3154` for information about improvements brought by
+  protocol 4.
 
 .. note::
    Serialization is a more primitive notion than persistence; although
@@ -179,8 +179,16 @@ The :mod:`pickle` module provides the following constants:
 
    An integer, the default :ref:`protocol version <pickle-protocols>` used
    for pickling.  May be less than :data:`HIGHEST_PROTOCOL`.  Currently the
-   default protocol is 3, a new protocol designed for Python 3.
+   default protocol is 4, first introduced in Python 3.4 and incompatible
+   with previous versions.
 
+   .. versionchanged:: 3.0
+
+      The default protocol is 3.
+
+   .. versionchanged:: 3.8
+
+      The default protocol is 4.
 
 The :mod:`pickle` module provides the following functions to make the pickling
 process more convenient:

Doc/whatsnew/3.8.rst

@@ -94,6 +94,9 @@ Improved Modules
 Optimizations
 =============
 
+* The default protocol in the :mod:`pickle` module is now Protocol 4,
+  first introduced in Python 3.4.  It offers better performance and smaller
+  size compared to Protocol 3 available since Python 3.0.
 
 Build and C API Changes
 =======================

Lib/pickle.py

@@ -57,9 +57,9 @@
 HIGHEST_PROTOCOL = 4
 
 # The protocol we write by default.  May be less than HIGHEST_PROTOCOL.
-# We intentionally write a protocol that Python 2.x cannot read;
-# there are too many issues with that.
-DEFAULT_PROTOCOL = 3
+# Only bump this if the oldest still supported version of Python already
+# includes it.
+DEFAULT_PROTOCOL = 4
 
 class PickleError(Exception):
     """A common base class for the other pickling exceptions."""
@@ -376,8 +376,8 @@ def __init__(self, file, protocol=None, *, fix_imports=True):
 
         The optional *protocol* argument tells the pickler to use the
         given protocol; supported protocols are 0, 1, 2, 3 and 4.  The
-        default protocol is 3; a backward-incompatible protocol designed
-        for Python 3.
+        default protocol is 4. It was introduced in Python 3.4, it is
+        incompatible with previous versions.
 
         Specifying a negative protocol version selects the highest
         protocol version supported.  The higher the protocol used, the

Misc/NEWS.d/next/Library/2018-04-02-16-10-12.bpo-23403.KG7ADV.rst

@@ -0,0 +1,4 @@
+``DEFAULT_PROTOCOL`` in :mod:`pickle` was bumped to 4. Protocol 4 is
+described in :pep:`3154` and available since Python 3.4. It offers
+better performance and smaller size compared to protocol 3 introduced
+in Python 3.0.

Modules/_pickle.c

@@ -20,10 +20,12 @@ class _pickle.UnpicklerMemoProxy "UnpicklerMemoProxyObject *" "&UnpicklerMemoPro
 [clinic start generated code]*/
 /*[clinic end generated code: output=da39a3ee5e6b4b0d input=4b3e113468a58e6c]*/
 
-/* Bump this when new opcodes are added to the pickle protocol. */
+/* Bump HIGHEST_PROTOCOL when new opcodes are added to the pickle protocol.
+   Bump DEFAULT_PROTOCOL only when the oldest still supported version of Python
+   already includes it. */
 enum {
     HIGHEST_PROTOCOL = 4,
-    DEFAULT_PROTOCOL = 3
+    DEFAULT_PROTOCOL = 4
 };
 
 /* Pickle opcodes. These must be kept updated with pickle.py.
@@ -7176,8 +7178,9 @@ This is equivalent to ``Pickler(file, protocol).dump(obj)``, but may
 be more efficient.
 
 The optional *protocol* argument tells the pickler to use the given
-protocol supported protocols are 0, 1, 2, 3 and 4.  The default
-protocol is 3; a backward-incompatible protocol designed for Python 3.
+protocol; supported protocols are 0, 1, 2, 3 and 4.  The default
+protocol is 4. It was introduced in Python 3.4, it is incompatible
+with previous versions.
 
 Specifying a negative protocol version selects the highest protocol
 version supported.  The higher the protocol used, the more recent the
@@ -7196,7 +7199,7 @@ to map the new Python 3 names to the old module names used in Python
 static PyObject *
 _pickle_dump_impl(PyObject *module, PyObject *obj, PyObject *file,
                   PyObject *protocol, int fix_imports)
-/*[clinic end generated code: output=a4774d5fde7d34de input=830f8a64cef6f042]*/
+/*[clinic end generated code: output=a4774d5fde7d34de input=93f1408489a87472]*/
 {
     PicklerObject *pickler = _Pickler_New();
 
@@ -7236,7 +7239,8 @@ Return the pickled representation of the object as a bytes object.
 
 The optional *protocol* argument tells the pickler to use the given
 protocol; supported protocols are 0, 1, 2, 3 and 4.  The default
-protocol is 3; a backward-incompatible protocol designed for Python 3.
+protocol is 4. It was introduced in Python 3.4, it is incompatible
+with previous versions.
 
 Specifying a negative protocol version selects the highest protocol
 version supported.  The higher the protocol used, the more recent the
@@ -7250,7 +7254,7 @@ Python 2, so that the pickle data stream is readable with Python 2.
 static PyObject *
 _pickle_dumps_impl(PyObject *module, PyObject *obj, PyObject *protocol,
                    int fix_imports)
-/*[clinic end generated code: output=d75d5cda456fd261 input=293dbeda181580b7]*/
+/*[clinic end generated code: output=d75d5cda456fd261 input=b6efb45a7d19b5ab]*/
 {
     PyObject *result;
     PicklerObject *pickler = _Pickler_New();

Modules/clinic/_pickle.c.h

@@ -367,8 +367,9 @@ PyDoc_STRVAR(_pickle_dump__doc__,
 "be more efficient.\n"
 "\n"
 "The optional *protocol* argument tells the pickler to use the given\n"
-"protocol supported protocols are 0, 1, 2, 3 and 4.  The default\n"
-"protocol is 3; a backward-incompatible protocol designed for Python 3.\n"
+"protocol; supported protocols are 0, 1, 2, 3 and 4.  The default\n"
+"protocol is 4. It was introduced in Python 3.4, it is incompatible\n"
+"with previous versions.\n"
 "\n"
 "Specifying a negative protocol version selects the highest protocol\n"
 "version supported.  The higher the protocol used, the more recent the\n"
@@ -419,7 +420,8 @@ PyDoc_STRVAR(_pickle_dumps__doc__,
 "\n"
 "The optional *protocol* argument tells the pickler to use the given\n"
 "protocol; supported protocols are 0, 1, 2, 3 and 4.  The default\n"
-"protocol is 3; a backward-incompatible protocol designed for Python 3.\n"
+"protocol is 4. It was introduced in Python 3.4, it is incompatible\n"
+"with previous versions.\n"
 "\n"
 "Specifying a negative protocol version selects the highest protocol\n"
 "version supported.  The higher the protocol used, the more recent the\n"
@@ -560,4 +562,4 @@ _pickle_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=e995dd494045d876 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=6fc104b8299c82dd input=a9049054013a1b77]*/