[3.8] bpo-38270: Check for hash digest algorithms and avoid MD5 (pythonGH-16382) (pythonGH-16393)

Make it easier to run and test Python on systems with restrict crypto policies:

* add requires_hashdigest to test.support to check if a hash digest algorithm is available and working
* avoid MD5 in test_hmac
* replace MD5 with SHA256 in test_tarfile
* mark network tests that require MD5 for MD5-based digest auth or CRAM-MD5

https://bugs.python.org/issue38270
(cherry picked from commit c64a1a6)

Co-authored-by: Christian Heimes <[email protected]>

https://bugs.python.org/issue38270

Automerge-Triggered-By: @tiran

Author: miss-islington

Lib/test/support/__init__.py

@@ -11,6 +11,7 @@
 import fnmatch
 import functools
 import gc
+import hashlib
 import importlib
 import importlib.util
 import io
@@ -627,6 +628,27 @@ def wrapper(*args, **kw):
         return wrapper
     return decorator
 
+def requires_hashdigest(digestname):
+    """Decorator raising SkipTest if a hashing algorithm is not available
+
+    The hashing algorithm could be missing or blocked by a strict crypto
+    policy.
+
+    ValueError: [digital envelope routines: EVP_DigestInit_ex] disabled for FIPS
+    ValueError: unsupported hash type md4
+    """
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            try:
+                hashlib.new(digestname)
+            except ValueError:
+                raise unittest.SkipTest(
+                    f"hash digest '{digestname}' is not available."
+                )
+            return func(*args, **kwargs)
+        return wrapper
+    return decorator
 
 # Don't use "localhost", since resolving it uses the DNS under recent
 # Windows versions (see issue #18792).

Lib/test/test_hmac.py

@@ -4,6 +4,8 @@
 import unittest
 import warnings
 
+from test.support import requires_hashdigest
+
 
 def ignore_warning(func):
     @functools.wraps(func)
@@ -17,6 +19,7 @@ def wrapper(*args, **kwargs):
 
 class TestVectorsTestCase(unittest.TestCase):
 
+    @requires_hashdigest('md5')
     def test_md5_vectors(self):
         # Test the HMAC module against test vectors from the RFC.
 
@@ -63,6 +66,7 @@ def md5test(key, data, digest):
                  b"and Larger Than One Block-Size Data"),
                 "6f630fad67cda0ee1fb1f562db3aa53e")
 
+    @requires_hashdigest('sha1')
     def test_sha_vectors(self):
         def shatest(key, data, digest):
             h = hmac.HMAC(key, data, digestmod=hashlib.sha1)
@@ -230,23 +234,28 @@ def hmactest(key, data, hexdigests):
                                    '134676fb6de0446065c97440fa8c6a58',
                  })
 
+    @requires_hashdigest('sha224')
     def test_sha224_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha224, 'sha224', 28, 64)
 
+    @requires_hashdigest('sha256')
     def test_sha256_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha256, 'sha256', 32, 64)
 
+    @requires_hashdigest('sha384')
     def test_sha384_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha384, 'sha384', 48, 128)
 
+    @requires_hashdigest('sha512')
     def test_sha512_rfc4231(self):
         self._rfc4231_test_cases(hashlib.sha512, 'sha512', 64, 128)
 
+    @requires_hashdigest('sha256')
     def test_legacy_block_size_warnings(self):
         class MockCrazyHash(object):
             """Ain't no block_size attribute here."""
             def __init__(self, *args):
-                self._x = hashlib.sha1(*args)
+                self._x = hashlib.sha256(*args)
                 self.digest_size = self._x.digest_size
             def update(self, v):
                 self._x.update(v)
@@ -273,88 +282,94 @@ def test_with_digestmod_warning(self):
             self.assertEqual(h.hexdigest().upper(), digest)
 
 
+
 class ConstructorTestCase(unittest.TestCase):
 
+    expected = (
+        "6c845b47f52b3b47f6590c502db7825aad757bf4fadc8fa972f7cd2e76a5bdeb"
+    )
     @ignore_warning
+    @requires_hashdigest('sha256')
     def test_normal(self):
         # Standard constructor call.
-        failed = 0
         try:
-            h = hmac.HMAC(b"key")
+            hmac.HMAC(b"key", digestmod='sha256')
         except Exception:
             self.fail("Standard constructor call raised exception.")
 
     @ignore_warning
+    @requires_hashdigest('sha256')
     def test_with_str_key(self):
         # Pass a key of type str, which is an error, because it expects a key
         # of type bytes
         with self.assertRaises(TypeError):
-            h = hmac.HMAC("key")
+            h = hmac.HMAC("key", digestmod='sha256')
 
-    @ignore_warning
+    @requires_hashdigest('sha256')
     def test_dot_new_with_str_key(self):
         # Pass a key of type str, which is an error, because it expects a key
         # of type bytes
         with self.assertRaises(TypeError):
-            h = hmac.new("key")
+            h = hmac.HMAC("key", digestmod='sha256')
 
     @ignore_warning
+    @requires_hashdigest('sha256')
     def test_withtext(self):
         # Constructor call with text.
         try:
-            h = hmac.HMAC(b"key", b"hash this!")
+            h = hmac.HMAC(b"key", b"hash this!", digestmod='sha256')
         except Exception:
             self.fail("Constructor call with text argument raised exception.")
-        self.assertEqual(h.hexdigest(), '34325b639da4cfd95735b381e28cb864')
+        self.assertEqual(h.hexdigest(), self.expected)
 
+    @requires_hashdigest('sha256')
     def test_with_bytearray(self):
         try:
             h = hmac.HMAC(bytearray(b"key"), bytearray(b"hash this!"),
-                          digestmod="md5")
+                          digestmod="sha256")
         except Exception:
             self.fail("Constructor call with bytearray arguments raised exception.")
-        self.assertEqual(h.hexdigest(), '34325b639da4cfd95735b381e28cb864')
+            self.assertEqual(h.hexdigest(), self.expected)
 
+    @requires_hashdigest('sha256')
     def test_with_memoryview_msg(self):
         try:
-            h = hmac.HMAC(b"key", memoryview(b"hash this!"), digestmod="md5")
+            h = hmac.HMAC(b"key", memoryview(b"hash this!"), digestmod="sha256")
         except Exception:
             self.fail("Constructor call with memoryview msg raised exception.")
-        self.assertEqual(h.hexdigest(), '34325b639da4cfd95735b381e28cb864')
+            self.assertEqual(h.hexdigest(), self.expected)
 
+    @requires_hashdigest('sha256')
     def test_withmodule(self):
         # Constructor call with text and digest module.
         try:
-            h = hmac.HMAC(b"key", b"", hashlib.sha1)
+            h = hmac.HMAC(b"key", b"", hashlib.sha256)
         except Exception:
-            self.fail("Constructor call with hashlib.sha1 raised exception.")
+            self.fail("Constructor call with hashlib.sha256 raised exception.")
 
-class SanityTestCase(unittest.TestCase):
 
-    @ignore_warning
-    def test_default_is_md5(self):
-        # Testing if HMAC defaults to MD5 algorithm.
-        # NOTE: this whitebox test depends on the hmac class internals
-        h = hmac.HMAC(b"key")
-        self.assertEqual(h.digest_cons, hashlib.md5)
+class SanityTestCase(unittest.TestCase):
 
+    @requires_hashdigest('sha256')
     def test_exercise_all_methods(self):
         # Exercising all methods once.
         # This must not raise any exceptions
         try:
-            h = hmac.HMAC(b"my secret key", digestmod="md5")
+            h = hmac.HMAC(b"my secret key", digestmod="sha256")
             h.update(b"compute the hash of this text!")
             dig = h.digest()
             dig = h.hexdigest()
             h2 = h.copy()
         except Exception:
             self.fail("Exception raised during normal usage of HMAC class.")
 
+
 class CopyTestCase(unittest.TestCase):
 
+    @requires_hashdigest('sha256')
     def test_attributes(self):
         # Testing if attributes are of same type.
-        h1 = hmac.HMAC(b"key", digestmod="md5")
+        h1 = hmac.HMAC(b"key", digestmod="sha256")
         h2 = h1.copy()
         self.assertTrue(h1.digest_cons == h2.digest_cons,
             "digest constructors don't match.")
@@ -363,9 +378,10 @@ def test_attributes(self):
         self.assertEqual(type(h1.outer), type(h2.outer),
             "Types of outer don't match.")
 
+    @requires_hashdigest('sha256')
     def test_realcopy(self):
         # Testing if the copy method created a real copy.
-        h1 = hmac.HMAC(b"key", digestmod="md5")
+        h1 = hmac.HMAC(b"key", digestmod="sha256")
         h2 = h1.copy()
         # Using id() in case somebody has overridden __eq__/__ne__.
         self.assertTrue(id(h1) != id(h2), "No real copy of the HMAC instance.")
@@ -374,9 +390,10 @@ def test_realcopy(self):
         self.assertTrue(id(h1.outer) != id(h2.outer),
             "No real copy of the attribute 'outer'.")
 
+    @requires_hashdigest('sha256')
     def test_equality(self):
         # Testing if the copy has the same digests.
-        h1 = hmac.HMAC(b"key", digestmod="md5")
+        h1 = hmac.HMAC(b"key", digestmod="sha256")
         h1.update(b"some random text")
         h2 = h1.copy()
         self.assertEqual(h1.digest(), h2.digest(),

Lib/test/test_imaplib.py

@@ -14,7 +14,8 @@
 import inspect
 
 from test.support import (reap_threads, verbose, transient_internet,
-                          run_with_tz, run_with_locale, cpython_only)
+                          run_with_tz, run_with_locale, cpython_only,
+                          requires_hashdigest)
 import unittest
 from unittest import mock
 from datetime import datetime, timezone, timedelta
@@ -369,6 +370,7 @@ def cmd_AUTHENTICATE(self, tag, args):
         self.assertEqual(code, 'OK')
         self.assertEqual(server.response, b'ZmFrZQ==\r\n')  # b64 encoded 'fake'
 
+    @requires_hashdigest('md5')
     def test_login_cram_md5_bytes(self):
         class AuthHandler(SimpleIMAPHandler):
             capabilities = 'LOGINDISABLED AUTH=CRAM-MD5'
@@ -386,6 +388,7 @@ def cmd_AUTHENTICATE(self, tag, args):
         ret, _ = client.login_cram_md5("tim", b"tanstaaftanstaaf")
         self.assertEqual(ret, "OK")
 
+    @requires_hashdigest('md5')
     def test_login_cram_md5_plain_text(self):
         class AuthHandler(SimpleIMAPHandler):
             capabilities = 'LOGINDISABLED AUTH=CRAM-MD5'
@@ -797,6 +800,7 @@ def cmd_AUTHENTICATE(self, tag, args):
                              b'ZmFrZQ==\r\n')  # b64 encoded 'fake'
 
     @reap_threads
+    @requires_hashdigest('md5')
     def test_login_cram_md5(self):
 
         class AuthHandler(SimpleIMAPHandler):

Lib/test/test_poplib.py

@@ -304,9 +304,11 @@ def test_noop(self):
     def test_rpop(self):
         self.assertOK(self.client.rpop('foo'))
 
+    @test_support.requires_hashdigest('md5')
     def test_apop_normal(self):
         self.assertOK(self.client.apop('foo', 'dummypassword'))
 
+    @test_support.requires_hashdigest('md5')
     def test_apop_REDOS(self):
         # Replace welcome with very long evil welcome.
         # NB The upper bound on welcome length is currently 2048.

Lib/test/test_smtplib.py

@@ -4,6 +4,7 @@
 from email.message import EmailMessage
 from email.base64mime import body_encode as encode_base64
 import email.utils
+import hashlib
 import hmac
 import socket
 import smtpd
@@ -18,6 +19,7 @@
 
 import unittest
 from test import support, mock_socket
+from test.support import requires_hashdigest
 from unittest.mock import Mock
 
 HOST = "localhost"
@@ -968,6 +970,7 @@ def testAUTH_LOGIN(self):
         self.assertEqual(resp, (235, b'Authentication Succeeded'))
         smtp.close()
 
+    @requires_hashdigest('md5')
     def testAUTH_CRAM_MD5(self):
         self.serv.add_feature("AUTH CRAM-MD5")
         smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15)
@@ -984,7 +987,13 @@ def testAUTH_multiple(self):
         smtp.close()
 
     def test_auth_function(self):
-        supported = {'CRAM-MD5', 'PLAIN', 'LOGIN'}
+        supported = {'PLAIN', 'LOGIN'}
+        try:
+            hashlib.md5()
+        except ValueError:
+            pass
+        else:
+            supported.add('CRAM-MD5')
         for mechanism in supported:
             self.serv.add_feature("AUTH {}".format(mechanism))
         for mechanism in supported: