bpo-33618: Enable TLS 1.3 in tests

TLS 1.3 behaves slightly different than TLS 1.2. Session tickets and TLS
client cert auth are now handled after the initialy handshake. Tests now
either send/recv data to trigger session and client certs. Or tests
ignore ConnectionResetError / BrokenPipeError on the server side to
handle clients that force-close the socket fd.

To test TLS 1.3, OpenSSL 1.1.1-pre7-dev (git master + OpenSSL PR
openssl/openssl#6340) is required.

Signed-off-by: Christian Heimes <[email protected]>

Author: tiran

Doc/library/ssl.rst

@@ -2587,7 +2587,33 @@ successful call of :func:`~ssl.RAND_add`, :func:`~ssl.RAND_bytes` or
 :func:`~ssl.RAND_pseudo_bytes` is sufficient.
 
 
-.. ssl-libressl:
+.. _ssl-tlsv1_3:
+
+TLS 1.3
+-------
+
+.. versionadded:: 3.7
+
+Python has provisional and experimental support for TLS 1.3 with OpenSSL
+1.1.1.  The new protocol behaves slightly differently than previous version
+of TLS/SSL.  Some new TLS 1.3 features are not yet available.
+
+- TLS 1.3 uses a disjunct set of cipher suites. All AES-GCM and
+  ChaCha20 cipher suites are enabled by default.  The method
+  :meth:`SSLContext.set_ciphers` cannot enable or disable any TLS 1.3
+  ciphers yet, but :meth:`SSLContext.get_cipers` returns them.
+- Session tickets are no longer sent as part of the initial handshake and
+  are handled differently.  :attr:`SSLSocket.session` and :class:`SSLSession`
+  are not compatible with TLS 1.3.
+- Client-side certificates are also no longer verified during the initial
+  handshake.  A server can request a certificate at any time.  Clients
+  process certificate requests while they send or receive application data
+  from the server.
+- TLS 1.3 features like early data, deferred TLS client cert request,
+  signature algorithm configuration, and rekeying are not supported yet.
+
+
+.. _ssl-libressl:
 
 LibreSSL support
 ----------------

Doc/whatsnew/3.7.rst

@@ -1244,8 +1244,8 @@ Host name validation can be customized with
 .. note::
    The improved host name check requires a *libssl* implementation compatible
    with OpenSSL 1.0.2 or 1.1.  Consequently, OpenSSL 0.9.8 and 1.0.1 are no
-   longer supported and LibreSSL is temporarily not supported until it gains
-   the necessary OpenSSL 1.0.2 APIs.
+   longer supported.  The ssl module is mostly compatible with LibreSSL 2.7.2
+   and newer.
 
 The ``ssl`` module no longer sends IP addresses in SNI TLS extension.
 (Contributed by Christian Heimes in :issue:`32185`.)
@@ -1270,8 +1270,12 @@ rather than the U-label form (``"pythön.org"``).  (Contributed by
 Nathaniel J. Smith and Christian Heimes in :issue:`28414`.)
 
 The ``ssl`` module has preliminary and experimental support for TLS 1.3 and
-OpenSSL 1.1.1.  (Contributed by Christian Heimes in :issue:`32947`,
-:issue:`20995`, :issue:`29136`, and :issue:`30622`)
+OpenSSL 1.1.1.  At the time of Python 3.7.0 release, OpenSSL 1.1.1 is still
+under development and TLS 1.3 hasn't been finalized yet.  The TLS 1.3
+handshake and protocol behaves slightly differently than TLS 1.2 and earlier,
+see :ref:`ssl-tlsv1_3`.
+(Contributed by Christian Heimes in :issue:`32947`, :issue:`20995`,
+:issue:`29136`, :issue:`30622` and :issue:`33618`)
 
 :class:`~ssl.SSLSocket` and :class:`~ssl.SSLObject` no longer have a public
 constructor.  Direct instantiation was never a documented and supported

Lib/test/test_asyncio/test_sslproto.py

@@ -251,6 +251,8 @@ def test_start_tls_server_1(self):
 
         server_context = test_utils.simple_server_sslcontext()
         client_context = test_utils.simple_client_sslcontext()
+        # TODO: fix TLSv1.3 support
+        client_context.options |= ssl.OP_NO_TLSv1_3
 
         def client(sock, addr):
             time.sleep(0.5)

Lib/test/test_asyncio/utils.py

@@ -74,8 +74,6 @@ def simple_server_sslcontext():
     server_context.load_cert_chain(ONLYCERT, ONLYKEY)
     server_context.check_hostname = False
     server_context.verify_mode = ssl.CERT_NONE
-    # TODO: fix TLSv1.3 support
-    server_context.options |= ssl.OP_NO_TLSv1_3
     return server_context
 
 

Lib/test/test_ftplib.py

@@ -257,6 +257,7 @@ class DummyFTPServer(asyncore.dispatcher, threading.Thread):
     def __init__(self, address, af=socket.AF_INET):
         threading.Thread.__init__(self)
         asyncore.dispatcher.__init__(self)
+        self.daemon = True
         self.create_socket(af, socket.SOCK_STREAM)
         self.bind(address)
         self.listen(5)
@@ -312,8 +313,6 @@ class SSLConnection(asyncore.dispatcher):
 
         def secure_connection(self):
             context = ssl.SSLContext()
-            # TODO: fix TLSv1.3 support
-            context.options |= ssl.OP_NO_TLSv1_3
             context.load_cert_chain(CERTFILE)
             socket = context.wrap_socket(self.socket,
                                          suppress_ragged_eofs=False,
@@ -405,7 +404,7 @@ def handle_error(self):
 
         def close(self):
             if (isinstance(self.socket, ssl.SSLSocket) and
-                self.socket._sslobj is not None):
+                    self.socket._sslobj is not None):
                 self._do_ssl_shutdown()
             else:
                 super(SSLConnection, self).close()
@@ -910,8 +909,6 @@ def test_auth_issued_twice(self):
     def test_context(self):
         self.client.quit()
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
-        # TODO: fix TLSv1.3 support
-        ctx.options |= ssl.OP_NO_TLSv1_3
         ctx.check_hostname = False
         ctx.verify_mode = ssl.CERT_NONE
         self.assertRaises(ValueError, ftplib.FTP_TLS, keyfile=CERTFILE,
@@ -944,8 +941,6 @@ def test_ccc(self):
     def test_check_hostname(self):
         self.client.quit()
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
-        # TODO: fix TLSv1.3 support
-        ctx.options |= ssl.OP_NO_TLSv1_3
         self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED)
         self.assertEqual(ctx.check_hostname, True)
         ctx.load_verify_locations(CAFILE)
@@ -982,6 +977,7 @@ def setUp(self):
         self.sock.settimeout(20)
         self.port = support.bind_port(self.sock)
         self.server_thread = threading.Thread(target=self.server)
+        self.server_thread.daemon = True
         self.server_thread.start()
         # Wait for the server to be ready.
         self.evt.wait()

Lib/test/test_poplib.py

@@ -153,8 +153,6 @@ def cmd_stls(self, arg):
             if self.tls_active is False:
                 self.push('+OK Begin TLS negotiation')
                 context = ssl.SSLContext()
-                # TODO: fix TLSv1.3 support
-                context.options |= ssl.OP_NO_TLSv1_3
                 context.load_cert_chain(CERTFILE)
                 tls_sock = context.wrap_socket(self.socket,
                                                server_side=True,
@@ -206,6 +204,7 @@ class DummyPOP3Server(asyncore.dispatcher, threading.Thread):
     def __init__(self, address, af=socket.AF_INET):
         threading.Thread.__init__(self)
         asyncore.dispatcher.__init__(self)
+        self.daemon = True
         self.create_socket(af, socket.SOCK_STREAM)
         self.bind(address)
         self.listen(5)
@@ -370,8 +369,6 @@ def test_stls(self):
     def test_stls_context(self):
         expected = b'+OK Begin TLS negotiation'
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
-        # TODO: fix TLSv1.3 support
-        ctx.options |= ssl.OP_NO_TLSv1_3
         ctx.load_verify_locations(CAFILE)
         self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED)
         self.assertEqual(ctx.check_hostname, True)
@@ -412,8 +409,6 @@ def test__all__(self):
 
     def test_context(self):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
-        # TODO: fix TLSv1.3 support
-        ctx.options |= ssl.OP_NO_TLSv1_3
         ctx.check_hostname = False
         ctx.verify_mode = ssl.CERT_NONE
         self.assertRaises(ValueError, poplib.POP3_SSL, self.server.host,
@@ -482,7 +477,7 @@ def setUp(self):
         self.sock.settimeout(60)  # Safety net. Look issue 11812
         self.port = test_support.bind_port(self.sock)
         self.thread = threading.Thread(target=self.server, args=(self.evt,self.sock))
-        self.thread.setDaemon(True)
+        self.thread.daemon = True
         self.thread.start()
         self.evt.wait()
 

Lib/test/test_ssl.py

@@ -1826,6 +1826,7 @@ def test_connect_capath(self):
             s.connect(self.server_addr)
             cert = s.getpeercert()
             self.assertTrue(cert)
+
         # Same with a bytes `capath` argument
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS)
         ctx.verify_mode = ssl.CERT_REQUIRED
@@ -1841,8 +1842,6 @@ def test_connect_cadata(self):
         der = ssl.PEM_cert_to_DER_cert(pem)
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS)
         ctx.verify_mode = ssl.CERT_REQUIRED
-        # TODO: fix TLSv1.3 support
-        ctx.options |= ssl.OP_NO_TLSv1_3
         ctx.load_verify_locations(cadata=pem)
         with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s:
             s.connect(self.server_addr)
@@ -1852,8 +1851,6 @@ def test_connect_cadata(self):
         # same with DER
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS)
         ctx.verify_mode = ssl.CERT_REQUIRED
-        # TODO: fix TLSv1.3 support
-        ctx.options |= ssl.OP_NO_TLSv1_3
         ctx.load_verify_locations(cadata=der)
         with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s:
             s.connect(self.server_addr)
@@ -2109,11 +2106,21 @@ def wrap_conn(self):
                     self.sock, server_side=True)
                 self.server.selected_npn_protocols.append(self.sslconn.selected_npn_protocol())
                 self.server.selected_alpn_protocols.append(self.sslconn.selected_alpn_protocol())
-            except (ssl.SSLError, ConnectionResetError, OSError) as e:
+            except (ConnectionResetError, BrokenPipeError) as e:
                 # We treat ConnectionResetError as though it were an
                 # SSLError - OpenSSL on Ubuntu abruptly closes the
                 # connection when asked to use an unsupported protocol.
                 #
+                # BrokenPipeError is raised in TLS 1.3 mode, when OpenSSL
+                # tries to send session tickets after handshake.
+                # https://github.com/openssl/openssl/issues/6342
+                self.server.conn_errors.append(str(e))
+                if self.server.chatty:
+                    handle_error("\n server:  bad connection attempt from " + repr(self.addr) + ":\n")
+                self.running = False
+                self.close()
+                return False
+            except (ssl.SSLError, OSError) as e:
                 # OSError may occur with wrong protocols, e.g. both
                 # sides use PROTOCOL_TLS_SERVER.
                 #
@@ -2220,11 +2227,22 @@ def run(self):
                             sys.stdout.write(" server: read %r (%s), sending back %r (%s)...\n"
                                              % (msg, ctype, msg.lower(), ctype))
                         self.write(msg.lower())
+                except ConnectionResetError:
+                    # XXX: OpenSSL 1.1.1 sometimes raises ConnectionResetError
+                    # when connection is not shut down gracefully.
+                    if self.server.chatty and support.verbose:
+                        sys.stdout.write(
+                            " Connection reset by peer: {}\n".format(
+                                self.addr)
+                        )
+                    self.close()
+                    self.running = False
                 except OSError:
                     if self.server.chatty:
                         handle_error("Test server failure:\n")
                     self.close()
                     self.running = False
+
                     # normally, we'd just stop here, but for the test
                     # harness, we want to stop the server
                     self.server.stop()
@@ -2299,6 +2317,11 @@ def run(self):
                 pass
             except KeyboardInterrupt:
                 self.stop()
+            except BaseException as e:
+                if support.verbose and self.chatty:
+                    sys.stdout.write(
+                        ' connection handling failed: ' + repr(e) + '\n')
+
         self.sock.close()
 
     def stop(self):
@@ -2745,8 +2768,6 @@ def test_check_hostname_idn(self):
 
         server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
         server_context.load_cert_chain(IDNSANSFILE)
-        # TODO: fix TLSv1.3 support
-        server_context.options |= ssl.OP_NO_TLSv1_3
 
         context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
         context.verify_mode = ssl.CERT_REQUIRED
@@ -2797,7 +2818,7 @@ def test_check_hostname_idn(self):
                 with self.assertRaises(ssl.CertificateError):
                     s.connect((HOST, server.port))
 
-    def test_wrong_cert(self):
+    def test_wrong_cert_tls12(self):
         """Connecting when the server rejects the client's certificate
 
         Launch a server with CERT_REQUIRED, and check that trying to
@@ -2808,9 +2829,8 @@ def test_wrong_cert(self):
         client_context.load_cert_chain(WRONG_CERT)
         # require TLS client authentication
         server_context.verify_mode = ssl.CERT_REQUIRED
-        # TODO: fix TLSv1.3 support
-        # With TLS 1.3, test fails with exception in server thread
-        server_context.options |= ssl.OP_NO_TLSv1_3
+        # TLS 1.3 has different handshake
+        client_context.maximum_version = ssl.TLSVersion.TLSv1_2
 
         server = ThreadedEchoServer(
             context=server_context, chatty=True, connectionchatty=True,
@@ -2835,6 +2855,36 @@ def test_wrong_cert(self):
             else:
                 self.fail("Use of invalid cert should have failed!")
 
+    @unittest.skipUnless(ssl.HAS_TLSv1_3, "Test needs TLS 1.3")
+    def test_wrong_cert_tls13(self):
+        client_context, server_context, hostname = testing_context()
+        client_context.load_cert_chain(WRONG_CERT)
+        server_context.verify_mode = ssl.CERT_REQUIRED
+        server_context.minimum_version = ssl.TLSVersion.TLSv1_3
+        client_context.minimum_version = ssl.TLSVersion.TLSv1_3
+
+        server = ThreadedEchoServer(
+            context=server_context, chatty=True, connectionchatty=True,
+        )
+        with server, \
+             client_context.wrap_socket(socket.socket(),
+                                        server_hostname=hostname) as s:
+            # TLS 1.3 perform client cert exchange after handshake
+            s.connect((HOST, server.port))
+            try:
+                s.write(b'data')
+                s.read(4)
+            except ssl.SSLError as e:
+                if support.verbose:
+                    sys.stdout.write("\nSSLError is %r\n" % e)
+            except OSError as e:
+                if e.errno != errno.ECONNRESET:
+                    raise
+                if support.verbose:
+                    sys.stdout.write("\nsocket.error is %r\n" % e)
+            else:
+                self.fail("Use of invalid cert should have failed!")
+
     def test_rude_shutdown(self):
         """A brutal shutdown of an SSL server should raise an OSError
         in the client when attempting handshake.
@@ -3405,14 +3455,16 @@ def serve():
             # Block on the accept and wait on the connection to close.
             evt.set()
             remote, peer = server.accept()
-            remote.recv(1)
+            remote.send(remote.recv(4))
 
         t = threading.Thread(target=serve)
         t.start()
         # Client wait until server setup and perform a connect.
         evt.wait()
         client = context.wrap_socket(socket.socket())
         client.connect((host, port))
+        client.send(b'data')
+        client.recv()
         client_addr = client.getsockname()
         client.close()
         t.join()
@@ -3465,7 +3517,7 @@ def test_version_basic(self):
                 self.assertIs(s.version(), None)
                 self.assertIs(s._sslobj, None)
                 s.connect((HOST, server.port))
-                if ssl.OPENSSL_VERSION_INFO >= (1, 1, 1):
+                if IS_OPENSSL_1_1_1 and ssl.HAS_TLSv1_3:
                     self.assertEqual(s.version(), 'TLSv1.3')
                 elif ssl.OPENSSL_VERSION_INFO >= (1, 0, 2):
                     self.assertEqual(s.version(), 'TLSv1.2')
@@ -3574,8 +3626,6 @@ def test_tls_unique_channel_binding(self):
             sys.stdout.write("\n")
 
         client_context, server_context, hostname = testing_context()
-        # TODO: fix TLSv1.3 support
-        client_context.options |= ssl.OP_NO_TLSv1_3
 
         server = ThreadedEchoServer(context=server_context,
                                     chatty=True,
@@ -3594,7 +3644,10 @@ def test_tls_unique_channel_binding(self):
 
                 # check if it is sane
                 self.assertIsNotNone(cb_data)
-                self.assertEqual(len(cb_data), 12) # True for TLSv1
+                if s.version() == 'TLSv1.3':
+                    self.assertEqual(len(cb_data), 48)
+                else:
+                    self.assertEqual(len(cb_data), 12)  # True for TLSv1
 
                 # and compare with the peers version
                 s.write(b"CB tls-unique\n")
@@ -3616,7 +3669,10 @@ def test_tls_unique_channel_binding(self):
                 # is it really unique
                 self.assertNotEqual(cb_data, new_cb_data)
                 self.assertIsNotNone(cb_data)
-                self.assertEqual(len(cb_data), 12) # True for TLSv1
+                if s.version() == 'TLSv1.3':
+                    self.assertEqual(len(cb_data), 48)
+                else:
+                    self.assertEqual(len(cb_data), 12)  # True for TLSv1
                 s.write(b"CB tls-unique\n")
                 peer_data_repr = s.read().strip()
                 self.assertEqual(peer_data_repr,

Misc/NEWS.d/next/Library/2018-05-23-20-14-34.bpo-33618.xU39lr.rst

@@ -0,0 +1,2 @@
+Finalize and document preliminary and experimental TLS 1.3 support with
+OpenSSL 1.1.1