Skip to content

gh-131298: eliminate HACL* static libraries for cryptographic modules #132438

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gpshead merged 14 commits into from
Apr 20, 2025
Merged

gh-131298: eliminate HACL* static libraries for cryptographic modules #132438

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
ac1180f
simplify HACL* build for MD5, SHA1, SHA2 and SHA3 modules
picnixz Apr 12, 2025
3f0b41e
remove statically linked libraries for HACL* implementation
picnixz Apr 13, 2025
08f207b
is it better now?
picnixz Apr 13, 2025
3a3fc29
is it better now?
picnixz Apr 13, 2025
a5d39fa
fixup
picnixz Apr 13, 2025
d6ff1d0
Merge remote-tracking branch 'upstream/main' into fix/hacl/dynamic-li…
picnixz Apr 16, 2025
de0232a
Present HACL* as a static or shared library.
picnixz Apr 16, 2025
c0b00bf
amend whitespace
picnixz Apr 18, 2025
da10142
remove temporary .so file as it requires more symlinks
picnixz Apr 18, 2025
6a47f8a
avoid smelly symbols
picnixz Apr 19, 2025
ca584d9
fixup checksums
picnixz Apr 19, 2025
f260cbb
regen sbom
picnixz Apr 19, 2025
7f9868c
fixup shell warnings and comments
picnixz Apr 19, 2025
c87bb27
it *should* work
picnixz Apr 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
remove statically linked libraries for HACL* implementation
  • Loading branch information
@picnixz
picnixz committed Apr 13, 2025
commit 3f0b41e86076763726e63e1cb640eb0e13687867
58 changes: 41 additions & 17 deletions Makefile.pre.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,13 +227,13 @@ ENSUREPIP= @ENSUREPIP@
# Internal static libraries
LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a
LIBEXPAT_A= Modules/expat/libexpat.a
LIBHACL_BLAKE2_A= Modules/_hacl/libHacl_Hash_Blake2.a
LIBHACL_HMAC_A= Modules/_hacl/libHacl_HMAC.a
LIBHACL_CFLAGS=@LIBHACL_CFLAGS@
LIBHACL_SIMD128_FLAGS=@LIBHACL_SIMD128_FLAGS@
LIBHACL_SIMD256_FLAGS=@LIBHACL_SIMD256_FLAGS@
LIBHACL_SIMD128_OBJS=@LIBHACL_SIMD128_OBJS@
LIBHACL_SIMD128_SRCS=$(LIBHACL_SIMD128_OBJS:.o=.c)
LIBHACL_SIMD256_OBJS=@LIBHACL_SIMD256_OBJS@
LIBHACL_SIMD256_SRCS=$(LIBHACL_SIMD256_OBJS:.o=.c)

# Module state, compiler flags and linker flags
# Empty CFLAGS and LDFLAGS are omitted.
Expand Down Expand Up @@ -657,18 +657,32 @@ LIBEXPAT_HEADERS= \
##########################################################################
# hashlib's HACL* library

LIBHACL_MD5_SRCS= \
Modules/_hacl/Hacl_Hash_MD5.c
LIBHACL_MD5_OBJS= \
Modules/_hacl/Hacl_Hash_MD5.o

LIBHACL_SHA1_SRCS= \
Modules/_hacl/Hacl_Hash_SHA1.c
LIBHACL_SHA1_OBJS= \
Modules/_hacl/Hacl_Hash_SHA1.o

LIBHACL_SHA2_SRCS= \
Modules/_hacl/Hacl_Hash_SHA2.c
LIBHACL_SHA2_OBJS= \
Modules/_hacl/Hacl_Hash_SHA2.o

LIBHACL_SHA3_SRCS= \
Modules/_hacl/Hacl_Hash_SHA3.c
LIBHACL_SHA3_OBJS= \
Modules/_hacl/Hacl_Hash_SHA3.o

LIBHACL_BLAKE2_SRCS= \
Modules/_hacl/Hacl_Hash_Blake2s.c \
Modules/_hacl/Hacl_Hash_Blake2b.c \
Modules/_hacl/Lib_Memzero0.c \
$(LIBHACL_SIMD128_SRCS) \
$(LIBHACL_SIMD256_SRCS)
LIBHACL_BLAKE2_OBJS= \
Modules/_hacl/Hacl_Hash_Blake2s.o \
Modules/_hacl/Hacl_Hash_Blake2b.o \
Expand Down Expand Up @@ -1460,7 +1474,25 @@ $(LIBEXPAT_A): $(LIBEXPAT_OBJS)
# Build HACL* static libraries for hashlib and HACL* HMAC.
#
# The contents of libHacl_Blake2.a vary depending on whether we
# have the ability to compile vectorized versions
# have the ability to compile vectorized versions.
#
# The HACL* modules are dynamically compiled and linked with the
# corresponding CPython built-in modules on demand, depending on
# whether the module is built or not. Since HMAC however requires
# them to be built, we need rules that are always enabled and that
# are not overwritten by those automatically deduced by makesetup.

Modules/_hacl/Hacl_Hash_MD5.o: $(srcdir)/Modules/_hacl/Hacl_Hash_MD5.c $(LIBHACL_MD5_HEADERS)
$(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_MD5.c

Modules/_hacl/Hacl_Hash_SHA1.o: $(srcdir)/Modules/_hacl/Hacl_Hash_SHA1.c $(LIBHACL_SHA1_HEADERS)
$(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_SHA1.c

Modules/_hacl/Hacl_Hash_SHA2.o: $(srcdir)/Modules/_hacl/Hacl_Hash_SHA2.c $(LIBHACL_SHA2_HEADERS)
$(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_SHA2.c

Modules/_hacl/Hacl_Hash_SHA3.o: $(srcdir)/Modules/_hacl/Hacl_Hash_SHA3.c $(LIBHACL_SHA3_HEADERS)
$(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_SHA3.c

Modules/_hacl/Hacl_Hash_Blake2s.o: $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s.c $(LIBHACL_BLAKE2_HEADERS)
$(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Hash_Blake2s.c
Expand All @@ -1483,20 +1515,12 @@ Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.o: $(srcdir)/Modules/_hacl/Ha
Modules/_hacl/Lib_Memzero0.o: $(srcdir)/Modules/_hacl/Lib_Memzero0.c $(LIBHACL_BLAKE2_HEADERS)
$(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Lib_Memzero0.c

$(LIBHACL_BLAKE2_A): $(LIBHACL_BLAKE2_OBJS)
-rm -f $@
$(AR) $(ARFLAGS) $@ $(LIBHACL_BLAKE2_OBJS)

Modules/_hacl/Hacl_HMAC.o: $(srcdir)/Modules/_hacl/Hacl_HMAC.c $(LIBHACL_HMAC_HEADERS)
$(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_HMAC.c

Modules/_hacl/Hacl_Streaming_HMAC.o: $(srcdir)/Modules/_hacl/Hacl_Streaming_HMAC.c $(LIBHACL_HMAC_HEADERS)
$(CC) -Wno-unused-variable -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Streaming_HMAC.c

$(LIBHACL_HMAC_A): $(LIBHACL_HMAC_OBJS)
-rm -f $@
$(AR) $(ARFLAGS) $@ $(LIBHACL_HMAC_OBJS)

# create relative links from build/lib.platform/egg.so to Modules/egg.so
# pybuilddir.txt is created too late. We cannot use it in Makefile
# targets. ln --relative is not portable.
Expand Down Expand Up @@ -3270,12 +3294,12 @@ MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h @LIBMPDEC_INTERNAL@
MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@
MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h
MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h
MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_MD5_HEADERS) $(srcdir)/Modules/_hacl/Hacl_Hash_MD5.c
MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA1_HEADERS) $(srcdir)/Modules/_hacl/Hacl_Hash_SHA1.c
MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(srcdir)/Modules/_hacl/Hacl_Hash_SHA2.c
MODULE__SHA3_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA3_HEADERS) $(srcdir)/Modules/_hacl/Hacl_Hash_SHA3.c
MODULE__BLAKE2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_BLAKE2_HEADERS) $(LIBHACL_BLAKE2_A)
MODULE__HMAC_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HMAC_HEADERS) $(LIBHACL_HMAC_A)
MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_MD5_HEADERS) $(LIBHACL_MD5_OBJS)
MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA1_HEADERS) $(LIBHACL_SHA1_OBJS)
MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_OBJS)
MODULE__SHA3_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA3_HEADERS) $(LIBHACL_SHA3_OBJS)
MODULE__BLAKE2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_BLAKE2_HEADERS) $(LIBHACL_BLAKE2_OBJS)
MODULE__HMAC_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HMAC_HEADERS) $(LIBHACL_HMAC_OBJS)
MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c
MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h
MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h
Expand Down
18 changes: 11 additions & 7 deletions Modules/Setup.stdlib.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,13 +77,17 @@
@MODULE_READLINE_TRUE@readline readline.c

# hashing builtins, can be disabled with --without-builtin-hashlib-hashes
@MODULE__MD5_TRUE@_md5 md5module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_MD5.c -D_BSD_SOURCE -D_DEFAULT_SOURCE
@MODULE__SHA1_TRUE@_sha1 sha1module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA1.c -D_BSD_SOURCE -D_DEFAULT_SOURCE
@MODULE__SHA2_TRUE@_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA2.c -D_BSD_SOURCE -D_DEFAULT_SOURCE
@MODULE__SHA3_TRUE@_sha3 sha3module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA3.c -D_BSD_SOURCE -D_DEFAULT_SOURCE
@MODULE__BLAKE2_TRUE@_blake2 blake2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Hash_Blake2.a -D_BSD_SOURCE -D_DEFAULT_SOURCE

@MODULE__HMAC_TRUE@_hmac hmacmodule.c
# HACL* dependencies are compiled separately, but they should be linked
# against the module when needed. In order for the extension module to
# be correctly linked, we should however specify which *.o will be used
# (but for which no Makefile rule will be auto-generated).
@MODULE__MD5_TRUE@_md5 md5module.c -I$(srcdir)/Modules/_hacl/include $(LIBHACL_MD5_OBJS)
@MODULE__SHA1_TRUE@_sha1 sha1module.c -I$(srcdir)/Modules/_hacl/include $(LIBHACL_SHA1_OBJS)
@MODULE__SHA2_TRUE@_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include $(LIBHACL_SHA2_OBJS)
@MODULE__SHA3_TRUE@_sha3 sha3module.c -I$(srcdir)/Modules/_hacl/include $(LIBHACL_SHA3_OBJS)
@MODULE__BLAKE2_TRUE@_blake2 blake2module.c -I$(srcdir)/Modules/_hacl/include $(LIBHACL_BLAKE2_OBJS)

@MODULE__HMAC_TRUE@_hmac hmacmodule.c $(LIBHACL_HMAC_OBJS)

############################################################################
# XML and text
Expand Down
3 changes: 2 additions & 1 deletion Modules/makesetup
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' |
'*shared*') doconfig=no; continue;;
'*disabled*') doconfig=disabled; continue;;
esac
objs=
srcs=
cpps=
libs=
Expand Down Expand Up @@ -180,6 +181,7 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' |
\$\(*_LIBS\)) libs="$libs $arg";;
\$\(*_LDFLAGS\)) libs="$libs $arg";;
\$\(*_RPATH\)) libs="$libs $arg";;
\$\(*_OBJS\)) objs="$objs $arg";;
\$*) libs="$libs $arg"
cpps="$cpps $arg";;
*.*) echo 1>&2 "bad word $arg in $line"
Expand Down Expand Up @@ -224,7 +226,6 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' |
case $noobjects in
yes) continue;;
esac
objs=''
for src in $srcs
do
case $src in
Expand Down
2 changes: 1 addition & 1 deletion configure
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 1 addition & 3 deletions configure.ac
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7923,9 +7923,7 @@ AC_SUBST([LIBHACL_SIMD256_OBJS])
dnl HMAC builtin library does not need OpenSSL for now. In the future
dnl we might want to rely on OpenSSL EVP/NID interface or implement
dnl our own for algorithm resolution.
PY_STDLIB_MOD([_hmac], [], [],
[$LIBHACL_CFLAGS],
[$LIBHACL_CFLAGS Modules/_hacl/libHacl_HMAC.a])
PY_STDLIB_MOD([_hmac], [], [], [$LIBHACL_CFLAGS], [$LIBHACL_CFLAGS])

PY_STDLIB_MOD([_ctypes],
[], [test "$have_libffi" = yes],
Expand Down
Loading