Skip to content

Commit

Permalink
gh-99108: Refactor _sha256 & _sha512 into _sha2. (#101924)
Browse files Browse the repository at this point in the history
This merges their code. They're backed by the same single HACL* static library, having them be a single module simplifies maintenance.

This should unbreak the wasm enscripten builds that currently fail due to linking in --whole-archive mode and the HACL* library appearing twice.

Long unnoticed error fixed: _sha512.SHA384Type was doubly assigned and was actually SHA512Type. Nobody depends on those internal names.

Also rename LIBHACL_ make vars to LIBHACL_SHA2_ in preperation for other future HACL things.
  • Loading branch information
gpshead authored Feb 16, 2023
1 parent 89ac665 commit 0b13575
Show file tree
Hide file tree
Showing 18 changed files with 1,310 additions and 1,493 deletions.
12 changes: 6 additions & 6 deletions Lib/hashlib.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,13 +92,13 @@ def __get_builtin_constructor(name):
import _md5
cache['MD5'] = cache['md5'] = _md5.md5
elif name in {'SHA256', 'sha256', 'SHA224', 'sha224'}:
import _sha256
cache['SHA224'] = cache['sha224'] = _sha256.sha224
cache['SHA256'] = cache['sha256'] = _sha256.sha256
import _sha2
cache['SHA224'] = cache['sha224'] = _sha2.sha224
cache['SHA256'] = cache['sha256'] = _sha2.sha256
elif name in {'SHA512', 'sha512', 'SHA384', 'sha384'}:
import _sha512
cache['SHA384'] = cache['sha384'] = _sha512.sha384
cache['SHA512'] = cache['sha512'] = _sha512.sha512
import _sha2
cache['SHA384'] = cache['sha384'] = _sha2.sha384
cache['SHA512'] = cache['sha512'] = _sha2.sha512
elif name in {'blake2b', 'blake2s'}:
import _blake2
cache['blake2b'] = _blake2.blake2b
Expand Down
22 changes: 9 additions & 13 deletions Lib/test/test_hashlib.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
# Test hashlib module
#
# $Id$
# Test the hashlib module.
#
# Copyright (C) 2005-2010 Gregory P. Smith ([email protected])
# Licensed to PSF under a Contributor Agreement.
Expand Down Expand Up @@ -28,7 +26,6 @@
from http.client import HTTPException


# default builtin hash module
default_builtin_hashes = {'md5', 'sha1', 'sha256', 'sha512', 'sha3', 'blake2'}
# --with-builtin-hashlib-hashes override
builtin_hashes = sysconfig.get_config_var("PY_BUILTIN_HASHLIB_HASHES")
Expand Down Expand Up @@ -66,6 +63,7 @@ def get_fips_mode():
requires_blake2 = unittest.skipUnless(_blake2, 'requires _blake2')

# bpo-46913: Don't test the _sha3 extension on a Python UBSAN build
# TODO(gh-99108): Revisit this after _sha3 uses HACL*.
SKIP_SHA3 = support.check_sanitizer(ub=True)
requires_sha3 = unittest.skipUnless(not SKIP_SHA3, 'requires _sha3')

Expand Down Expand Up @@ -107,7 +105,7 @@ class HashLibTestCase(unittest.TestCase):

shakes = {'shake_128', 'shake_256'}

# Issue #14693: fallback modules are always compiled under POSIX
# gh-58898: Fallback modules are always compiled under POSIX.
_warn_on_extension_import = (os.name == 'posix' or support.Py_DEBUG)

def _conditional_import_module(self, module_name):
Expand All @@ -116,7 +114,7 @@ def _conditional_import_module(self, module_name):
return importlib.import_module(module_name)
except ModuleNotFoundError as error:
if self._warn_on_extension_import and module_name in builtin_hashes:
warnings.warn('Did a C extension fail to compile? %s' % error)
warnings.warn(f'Did a C extension fail to compile? {error}')
return None

def __init__(self, *args, **kwargs):
Expand Down Expand Up @@ -147,7 +145,7 @@ def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, **kwargs):
_hashlib = self._conditional_import_module('_hashlib')
self._hashlib = _hashlib
if _hashlib:
# These two algorithms should always be present when this module
# These algorithms should always be present when this module
# is compiled. If not, something was compiled wrong.
self.assertTrue(hasattr(_hashlib, 'openssl_md5'))
self.assertTrue(hasattr(_hashlib, 'openssl_sha1'))
Expand All @@ -172,12 +170,10 @@ def add_builtin_constructor(name):
_sha1 = self._conditional_import_module('_sha1')
if _sha1:
add_builtin_constructor('sha1')
_sha256 = self._conditional_import_module('_sha256')
if _sha256:
_sha2 = self._conditional_import_module('_sha2')
if _sha2:
add_builtin_constructor('sha224')
add_builtin_constructor('sha256')
_sha512 = self._conditional_import_module('_sha512')
if _sha512:
add_builtin_constructor('sha384')
add_builtin_constructor('sha512')
if _blake2:
Expand Down Expand Up @@ -460,9 +456,9 @@ def check_blocksize_name(self, name, block_size=0, digest_size=0,
self.assertEqual(len(m.hexdigest()), 2*digest_size)
self.assertEqual(m.name, name)
# split for sha3_512 / _sha3.sha3 object
self.assertIn(name.split("_")[0], repr(m))
self.assertIn(name.split("_")[0], repr(m).lower())

def test_blocksize_name(self):
def test_blocksize_and_name(self):
self.check_blocksize_name('md5', 64, 16)
self.check_blocksize_name('sha1', 64, 20)
self.check_blocksize_name('sha224', 64, 28)
Expand Down
15 changes: 7 additions & 8 deletions Makefile.pre.in
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ ENSUREPIP= @ENSUREPIP@
# Internal static libraries
LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a
LIBEXPAT_A= Modules/expat/libexpat.a
LIBHACL_A= Modules/_hacl/libHacl_Streaming_SHA2.a
LIBHACL_SHA2_A= Modules/_hacl/libHacl_Streaming_SHA2.a

# Module state, compiler flags and linker flags
# Empty CFLAGS and LDFLAGS are omitted.
Expand Down Expand Up @@ -575,10 +575,10 @@ LIBEXPAT_HEADERS= \
##########################################################################
# hashlib's HACL* library

LIBHACL_OBJS= \
LIBHACL_SHA2_OBJS= \
Modules/_hacl/Hacl_Streaming_SHA2.o

LIBHACL_HEADERS= \
LIBHACL_SHA2_HEADERS= \
Modules/_hacl/Hacl_Streaming_SHA2.h \
Modules/_hacl/include/krml/FStar_UInt128_Verified.h \
Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h \
Expand Down Expand Up @@ -912,12 +912,12 @@ $(LIBEXPAT_A): $(LIBEXPAT_OBJS)
# Build HACL* static libraries for hashlib: libHacl_Streaming_SHA2.a
LIBHACL_CFLAGS=-I$(srcdir)/Modules/_hacl/include -D_BSD_SOURCE -D_DEFAULT_SOURCE $(PY_STDMODULE_CFLAGS) $(CCSHARED)

Modules/_hacl/Hacl_Streaming_SHA2.o: $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.c $(LIBHACL_HEADERS)
Modules/_hacl/Hacl_Streaming_SHA2.o: $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.c $(LIBHACL_SHA2_HEADERS)
$(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.c

$(LIBHACL_A): $(LIBHACL_OBJS)
$(LIBHACL_SHA2_A): $(LIBHACL_SHA2_OBJS)
-rm -f $@
$(AR) $(ARFLAGS) $@ $(LIBHACL_OBJS)
$(AR) $(ARFLAGS) $@ $(LIBHACL_SHA2_OBJS)

# create relative links from build/lib.platform/egg.so to Modules/egg.so
# pybuilddir.txt is created too late. We cannot use it in Makefile
Expand Down Expand Up @@ -2635,9 +2635,8 @@ MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h
MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h
MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h
MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h
MODULE__SHA256_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) $(LIBHACL_A)
MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_A)
MODULE__SHA3_DEPS=$(srcdir)/Modules/_sha3/sha3.c $(srcdir)/Modules/_sha3/sha3.h $(srcdir)/Modules/hashlib.h
MODULE__SHA512_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) $(LIBHACL_A)
MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c
MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data.h $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h
MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/testcapi_long.h $(srcdir)/Modules/_testcapi/parts.h
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
The built-in extension modules for :mod:`hashlib` SHA2 algorithms, used when
OpenSSL does not provide them, now live in a single internal ``_sha2`` module
instead of separate ``_sha256`` and ``_sha512`` modules.
3 changes: 1 addition & 2 deletions Modules/Setup
Original file line number Diff line number Diff line change
Expand Up @@ -165,8 +165,7 @@ PYTHONPATH=$(COREPYTHONPATH)
#_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c
#_md5 md5module.c
#_sha1 sha1module.c
#_sha256 sha256module.c
#_sha512 sha512module.c
#_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a
#_sha3 _sha3/sha3module.c

# text encodings and unicode
Expand Down
3 changes: 1 addition & 2 deletions Modules/Setup.stdlib.in
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,7 @@
# hashing builtins, can be disabled with --without-builtin-hashlib-hashes
@MODULE__MD5_TRUE@_md5 md5module.c
@MODULE__SHA1_TRUE@_sha1 sha1module.c
@MODULE__SHA256_TRUE@_sha256 sha256module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a
@MODULE__SHA512_TRUE@_sha512 sha512module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a
@MODULE__SHA2_TRUE@_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a
@MODULE__SHA3_TRUE@_sha3 _sha3/sha3module.c
@MODULE__BLAKE2_TRUE@_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c

Expand Down
225 changes: 0 additions & 225 deletions Modules/clinic/sha256module.c.h

This file was deleted.

Loading

0 comments on commit 0b13575

Please sign in to comment.