Issue python#16113: Add SHA-3 and SHAKE support to hashlib module.

Doc/library/hashlib.rst

@@ -69,7 +69,13 @@ Constructors for hash algorithms that are always present in this module are
 :func:`md5` is normally available as well, though it
 may be missing if you are using a rare "FIPS compliant" build of Python.
 Additional algorithms may also be available depending upon the OpenSSL
-library that Python uses on your platform.
+library that Python uses on your platform. On most platforms the
+:func:`sha3_224`, :func:`sha3_256`, :func:`sha3_384`, :func:`sha3_512`,
+:func:`shake_128`, :func:`shake_256` are also available.
+
+.. versionadded:: 3.6
+   SHA3 (Keccak) and SHAKE constructors :func:`sha3_224`, :func:`sha3_256`,
+   :func:`sha3_384`, :func:`sha3_512`, :func:`shake_128`, :func:`shake_256`.
 
 .. versionadded:: 3.6
    :func:`blake2b` and :func:`blake2s` were added.
@@ -189,6 +195,28 @@ A hash object has the following methods:
    compute the digests of data sharing a common initial substring.
 
 
+SHAKE variable length digests
+-----------------------------
+
+The :func:`shake_128` and :func:`shake_256` algorithms provide variable
+length digests with length_in_bits//2 up to 128 or 256 bits of security.
+As such, their digest methods require a length. Maximum length is not limited
+by the SHAKE algorithm.
+
+.. method:: shake.digest(length)
+
+   Return the digest of the data passed to the :meth:`update` method so far.
+   This is a bytes object of size ``length`` which may contain bytes in
+   the whole range from 0 to 255.
+
+
+.. method:: shake.hexdigest(length)
+
+   Like :meth:`digest` except the digest is returned as a string object of
+   double length, containing only hexadecimal digits.  This may be used to
+   exchange the value safely in email or other non-binary environments.
+
+
 Key derivation
 --------------
 

Lib/hashlib.py

@@ -11,7 +11,8 @@
 Named constructor functions are also available, these are faster
 than using new(name):
 
-md5(), sha1(), sha224(), sha256(), sha384(), sha512(), blake2b(), and blake2s()
+md5(), sha1(), sha224(), sha256(), sha384(), sha512(), blake2b(), blake2s(),
+sha3_224, sha3_256, sha3_384, sha3_512, shake_128, and shake_256.
 
 More algorithms may be available on your platform but the above are guaranteed
 to exist.  See the algorithms_guaranteed and algorithms_available attributes
@@ -55,7 +56,10 @@
 # This tuple and __get_builtin_constructor() must be modified if a new
 # always available algorithm is added.
 __always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512',
-                      'blake2b', 'blake2s')
+                      'blake2b', 'blake2s',
+                      'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512',
+                      'shake_128', 'shake_256')
+
 
 algorithms_guaranteed = set(__always_supported)
 algorithms_available = set(__always_supported)
@@ -90,6 +94,15 @@ def __get_builtin_constructor(name):
             import _blake2
             cache['blake2b'] = _blake2.blake2b
             cache['blake2s'] = _blake2.blake2s
+        elif name in {'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512',
+                      'shake_128', 'shake_256'}:
+            import _sha3
+            cache['sha3_224'] = _sha3.sha3_224
+            cache['sha3_256'] = _sha3.sha3_256
+            cache['sha3_384'] = _sha3.sha3_384
+            cache['sha3_512'] = _sha3.sha3_512
+            cache['shake_128'] = _sha3.shake_128
+            cache['shake_256'] = _sha3.shake_256
     except ImportError:
         pass  # no extension module, this hash is unsupported.
 

Lib/test/test_hashlib.py

@@ -34,6 +34,13 @@
 
 requires_blake2 = unittest.skipUnless(_blake2, 'requires _blake2')
 
+try:
+    import _sha3
+except ImportError:
+    _sha3 = None
+
+requires_sha3 = unittest.skipUnless(_sha3, 'requires _sha3')
+
 
 def hexstr(s):
     assert isinstance(s, bytes), repr(s)
@@ -61,7 +68,11 @@ class HashLibTestCase(unittest.TestCase):
     supported_hash_names = ( 'md5', 'MD5', 'sha1', 'SHA1',
                              'sha224', 'SHA224', 'sha256', 'SHA256',
                              'sha384', 'SHA384', 'sha512', 'SHA512',
-                             'blake2b', 'blake2s')
+                             'blake2b', 'blake2s',
+                             'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512',
+                             'shake_128', 'shake_256')
+
+    shakes = {'shake_128', 'shake_256'}
 
     # Issue #14693: fallback modules are always compiled under POSIX
     _warn_on_extension_import = os.name == 'posix' or COMPILED_WITH_PYDEBUG
@@ -131,6 +142,15 @@ def add_builtin_constructor(name):
             add_builtin_constructor('blake2s')
             add_builtin_constructor('blake2b')
 
+        _sha3 = self._conditional_import_module('_sha3')
+        if _sha3:
+            add_builtin_constructor('sha3_224')
+            add_builtin_constructor('sha3_256')
+            add_builtin_constructor('sha3_384')
+            add_builtin_constructor('sha3_512')
+            add_builtin_constructor('shake_128')
+            add_builtin_constructor('shake_256')
+
         super(HashLibTestCase, self).__init__(*args, **kwargs)
 
     @property
@@ -142,7 +162,10 @@ def test_hash_array(self):
         a = array.array("b", range(10))
         for cons in self.hash_constructors:
             c = cons(a)
-            c.hexdigest()
+            if c.name in self.shakes:
+                c.hexdigest(16)
+            else:
+                c.hexdigest()
 
     def test_algorithms_guaranteed(self):
         self.assertEqual(hashlib.algorithms_guaranteed,
@@ -186,14 +209,21 @@ def test_get_builtin_constructor(self):
     def test_hexdigest(self):
         for cons in self.hash_constructors:
             h = cons()
-            self.assertIsInstance(h.digest(), bytes)
-            self.assertEqual(hexstr(h.digest()), h.hexdigest())
+            if h.name in self.shakes:
+                self.assertIsInstance(h.digest(16), bytes)
+                self.assertEqual(hexstr(h.digest(16)), h.hexdigest(16))
+            else:
+                self.assertIsInstance(h.digest(), bytes)
+                self.assertEqual(hexstr(h.digest()), h.hexdigest())
 
     def test_name_attribute(self):
         for cons in self.hash_constructors:
             h = cons()
             self.assertIsInstance(h.name, str)
-            self.assertIn(h.name, self.supported_hash_names)
+            if h.name in self.supported_hash_names:
+                self.assertIn(h.name, self.supported_hash_names)
+            else:
+                self.assertNotIn(h.name, self.supported_hash_names)
             self.assertEqual(h.name, hashlib.new(h.name).name)
 
     def test_large_update(self):
@@ -208,40 +238,46 @@ def test_large_update(self):
             m1.update(bees)
             m1.update(cees)
             m1.update(dees)
+            if m1.name in self.shakes:
+                args = (16,)
+            else:
+                args = ()
 
             m2 = cons()
             m2.update(aas + bees + cees + dees)
-            self.assertEqual(m1.digest(), m2.digest())
+            self.assertEqual(m1.digest(*args), m2.digest(*args))
 
             m3 = cons(aas + bees + cees + dees)
-            self.assertEqual(m1.digest(), m3.digest())
+            self.assertEqual(m1.digest(*args), m3.digest(*args))
 
             # verify copy() doesn't touch original
             m4 = cons(aas + bees + cees)
-            m4_digest = m4.digest()
+            m4_digest = m4.digest(*args)
             m4_copy = m4.copy()
             m4_copy.update(dees)
-            self.assertEqual(m1.digest(), m4_copy.digest())
-            self.assertEqual(m4.digest(), m4_digest)
+            self.assertEqual(m1.digest(*args), m4_copy.digest(*args))
+            self.assertEqual(m4.digest(*args), m4_digest)
 
-    def check(self, name, data, hexdigest, **kwargs):
+    def check(self, name, data, hexdigest, shake=False, **kwargs):
+        length = len(hexdigest)//2
         hexdigest = hexdigest.lower()
         constructors = self.constructors_to_test[name]
         # 2 is for hashlib.name(...) and hashlib.new(name, ...)
         self.assertGreaterEqual(len(constructors), 2)
         for hash_object_constructor in constructors:
             m = hash_object_constructor(data, **kwargs)
-            computed = m.hexdigest()
+            computed = m.hexdigest() if not shake else m.hexdigest(length)
             self.assertEqual(
                     computed, hexdigest,
                     "Hash algorithm %s constructed using %s returned hexdigest"
                     " %r for %d byte input data that should have hashed to %r."
                     % (name, hash_object_constructor,
                        computed, len(data), hexdigest))
-            computed = m.digest()
+            computed = m.digest() if not shake else m.digest(length)
             digest = bytes.fromhex(hexdigest)
             self.assertEqual(computed, digest)
-            self.assertEqual(len(digest), m.digest_size)
+            if not shake:
+                self.assertEqual(len(digest), m.digest_size)
 
     def check_no_unicode(self, algorithm_name):
         # Unicode objects are not allowed as input.
@@ -262,13 +298,30 @@ def test_no_unicode_blake2(self):
         self.check_no_unicode('blake2b')
         self.check_no_unicode('blake2s')
 
-    def check_blocksize_name(self, name, block_size=0, digest_size=0):
+    @requires_sha3
+    def test_no_unicode_sha3(self):
+        self.check_no_unicode('sha3_224')
+        self.check_no_unicode('sha3_256')
+        self.check_no_unicode('sha3_384')
+        self.check_no_unicode('sha3_512')
+        self.check_no_unicode('shake_128')
+        self.check_no_unicode('shake_256')
+
+    def check_blocksize_name(self, name, block_size=0, digest_size=0,
+                             digest_length=None):
         constructors = self.constructors_to_test[name]
         for hash_object_constructor in constructors:
             m = hash_object_constructor()
             self.assertEqual(m.block_size, block_size)
             self.assertEqual(m.digest_size, digest_size)
-            self.assertEqual(len(m.digest()), digest_size)
+            if digest_length:
+                self.assertEqual(len(m.digest(digest_length)),
+                                 digest_length)
+                self.assertEqual(len(m.hexdigest(digest_length)),
+                                 2*digest_length)
+            else:
+                self.assertEqual(len(m.digest()), digest_size)
+                self.assertEqual(len(m.hexdigest()), 2*digest_size)
             self.assertEqual(m.name, name)
             # split for sha3_512 / _sha3.sha3 object
             self.assertIn(name.split("_")[0], repr(m))
@@ -280,6 +333,12 @@ def test_blocksize_name(self):
         self.check_blocksize_name('sha256', 64, 32)
         self.check_blocksize_name('sha384', 128, 48)
         self.check_blocksize_name('sha512', 128, 64)
+        self.check_blocksize_name('sha3_224', 144, 28)
+        self.check_blocksize_name('sha3_256', 136, 32)
+        self.check_blocksize_name('sha3_384', 104, 48)
+        self.check_blocksize_name('sha3_512', 72, 64)
+        self.check_blocksize_name('shake_128', 168, 0, 32)
+        self.check_blocksize_name('shake_256', 136, 0, 64)
 
     @requires_blake2
     def test_blocksize_name_blake2(self):
@@ -563,6 +622,72 @@ def test_blake2s_vectors(self):
             key = bytes.fromhex(key)
             self.check('blake2s', msg, md, key=key)
 
+    @requires_sha3
+    def test_case_sha3_224_0(self):
+        self.check('sha3_224', b"",
+          "6b4e03423667dbb73b6e15454f0eb1abd4597f9a1b078e3f5b5a6bc7")
+
+    @requires_sha3
+    def test_case_sha3_224_vector(self):
+        for msg, md in read_vectors('sha3_224'):
+            self.check('sha3_224', msg, md)
+
+    @requires_sha3
+    def test_case_sha3_256_0(self):
+        self.check('sha3_256', b"",
+          "a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a")
+
+    @requires_sha3
+    def test_case_sha3_256_vector(self):
+        for msg, md in read_vectors('sha3_256'):
+            self.check('sha3_256', msg, md)
+
+    @requires_sha3
+    def test_case_sha3_384_0(self):
+        self.check('sha3_384', b"",
+          "0c63a75b845e4f7d01107d852e4c2485c51a50aaaa94fc61995e71bbee983a2a"+
+          "c3713831264adb47fb6bd1e058d5f004")
+
+    @requires_sha3
+    def test_case_sha3_384_vector(self):
+        for msg, md in read_vectors('sha3_384'):
+            self.check('sha3_384', msg, md)
+
+    @requires_sha3
+    def test_case_sha3_512_0(self):
+        self.check('sha3_512', b"",
+          "a69f73cca23a9ac5c8b567dc185a756e97c982164fe25859e0d1dcc1475c80a6"+
+          "15b2123af1f5f94c11e3e9402c3ac558f500199d95b6d3e301758586281dcd26")
+
+    @requires_sha3
+    def test_case_sha3_512_vector(self):
+        for msg, md in read_vectors('sha3_512'):
+            self.check('sha3_512', msg, md)
+
+    @requires_sha3
+    def test_case_shake_128_0(self):
+        self.check('shake_128', b"",
+          "7f9c2ba4e88f827d616045507605853ed73b8093f6efbc88eb1a6eacfa66ef26",
+          True)
+        self.check('shake_128', b"", "7f9c", True)
+
+    @requires_sha3
+    def test_case_shake128_vector(self):
+        for msg, md in read_vectors('shake_128'):
+            self.check('shake_128', msg, md, True)
+
+    @requires_sha3
+    def test_case_shake_256_0(self):
+        self.check('shake_256', b"",
+          "46b9dd2b0ba88d13233b3feb743eeb243fcd52ea62b81b82b50c27646ed5762f",
+          True)
+        self.check('shake_256', b"", "46b9", True)
+
+    @requires_sha3
+    def test_case_shake256_vector(self):
+        for msg, md in read_vectors('shake_256'):
+            self.check('shake_256', msg, md, True)
+
     def test_gil(self):
         # Check things work fine with an input larger than the size required
         # for multithreaded operation (which is hardwired to 2048).

Misc/NEWS

@@ -91,6 +91,8 @@ Core and Builtins
 Library
 -------
 
+- Issue #16113: Add SHA-3 and SHAKE support to hashlib module.
+
 - Issue #27776: The :func:`os.urandom` function does now block on Linux 3.17
   and newer until the system urandom entropy pool is initialized to increase
   the security. This change is part of the :pep:`524`.

Modules/_sha3/README.txt

@@ -0,0 +1,11 @@
+Keccak Code Package
+===================
+
+The files in kcp are taken from the Keccak Code Package. They have been
+slightly to be C89 compatible. The architecture specific header file
+KeccakP-1600-SnP.h ha been renamed to KeccakP-1600-SnP-opt32.h or
+KeccakP-1600-SnP-opt64.h.
+
+The 64bit files were generated with generic64lc/libkeccak.a.pack target, the
+32bit files with generic32lc/libkeccak.a.pack.
+