Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Streamline the processes of building libbls, testing it, and creating the blspy wheels #367

Merged
merged 1 commit into from
Jun 15, 2023
Merged

Streamline the processes of building libbls, testing it, and creating the blspy wheels #367

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 0 additions & 20 deletions .github/workflows/build-wheels.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,26 +154,6 @@ jobs:
CIBW_BEFORE_BUILD_MACOS: >
python -m pip install --upgrade pip
CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=10.14"
CIBW_BEFORE_ALL_WINDOWS: >
curl -L https://download.libsodium.org/libsodium/releases/libsodium-1.0.18-stable-msvc.zip > libsodium-1.0.18-stable-msvc.zip
&& 7z x libsodium-1.0.18-stable-msvc.zip
&& git clone https://github.com/Chia-Network/relic_ietf_64.git
&& ls -l relic_ietf_64
&& git clone https://github.com/Chia-Network/mpir_gc_x64.git
&& ls -l mpir_gc_x64
CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: >
ls -l mpir_gc_x64 && pip uninstall -y delocate
&& pip install git+https://github.com/Chia-Network/delocate.git
&& delocate-wheel -v -i mpir_gc_x64/mpir.dll {wheel}
&& delocate-wheel -v -i mpir_gc_x64/mpir_gc.dll {wheel}
&& delocate-wheel -v -i mpir_gc_x64/mpir_broadwell.dll {wheel}
&& delocate-wheel -v -i mpir_gc_x64/mpir_broadwell_avx.dll {wheel}
&& delocate-wheel -v -i mpir_gc_x64/mpir_bulldozer.dll {wheel}
&& delocate-wheel -v -i mpir_gc_x64/mpir_haswell.dll {wheel}
&& delocate-wheel -v -i mpir_gc_x64/mpir_piledriver.dll {wheel}
&& delocate-wheel -v -i mpir_gc_x64/mpir_sandybridge.dll {wheel}
&& delocate-wheel -v -i mpir_gc_x64/mpir_skylake_avx.dll {wheel}
&& cp {wheel} {dest_dir}
CIBW_TEST_REQUIRES: pytest
CIBW_TEST_COMMAND: py.test -v {project}/python-bindings/test.py
run:
Expand Down
34 changes: 29 additions & 5 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ if(DEFINED ENV{RELIC_MAIN})
set(RELIC_REPOSITORY "https://github.com/relic-toolkit/relic.git")
else()
# This is currently anchored to upstream aecdcae7956f542fbee2392c1f0feb0a8ac41dc5
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this comment looks out-dated now

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We're still anchored to upstream aecdcae.

set(RELIC_GIT_TAG "215c69966cb78b255995f0ee9c86bbbb41c3c42b")
set(RELIC_GIT_TAG "2a4ed8ded3a9a5ac5e39f3877f357264b2e16f11")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what changed here? We don't have any regression tests for the BLS functionality itself (afaik), so we should be careful when updating relic to ensure it's still running regression tests

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can check the changes here, and we run tests (just like upstream) in addition to our own bls-signatures tests so we should be reasonably careful (at least to the degree that our tests cover).

set(RELIC_REPOSITORY "https://github.com/Chia-Network/relic.git")
endif()

Expand All @@ -63,12 +63,25 @@ FetchContent_Declare(

set(STBIN "off" CACHE STRING "Relic - Build static binaries")

find_package(gmp)
if (GMP_FOUND)
message(STATUS "Found libgmp")
if(MSVC)
set(ARITH "gmp" CACHE STRING "")
# We'll be using our custom MPIR build instead of GMP for MSVC builds
FetchContent_Declare(
chia_mpir
GIT_REPOSITORY https://github.com/Chia-Network/mpir_gc_x64.git
GIT_SHALLOW true
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do we want to pin this to a specific commit?
I would think it would make for fewer moving parts as we can update our MPIR build independently of taking that update in the blspy build

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We're in control of this repo, we rarely update those DLLs and we would like to take the latest. Furthermore, having inconsistency in what dlls we fetch from this dependency, across repos, would result in issues.

)
FetchContent_MakeAvailable(chia_mpir)
set(MPIR_INCLUDE_DIR ${chia_mpir_SOURCE_DIR} CACHE STRING "")
set(MPIR_IMPLIB "${chia_mpir_SOURCE_DIR}/mpir.lib" CACHE STRING "")
else()
set(ARITH "easy" CACHE STRING "")
find_package(gmp)
if(GMP_FOUND)
message(STATUS "Found libgmp")
set(ARITH "gmp" CACHE STRING "")
else()
set(ARITH "easy" CACHE STRING "")
endif()
endif()

if(CMAKE_SIZEOF_VOID_P EQUAL 4)
Expand Down Expand Up @@ -141,6 +154,17 @@ set(PP_METHD "LAZYR;OATEP" CACHE STRING "")

FetchContent_MakeAvailable(relic)

function(handle_mpir_dlls _target)
file(GLOB chia_mpir_dlls ${chia_mpir_SOURCE_DIR}/*.dll)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

GLOB makes me nervous. Do we not know what the filename is of this DLL?
The reason it makes me nervous is that it makes the build less deterministic, it depends on what files happen to be on your filesystem. perhaps you have some old files left-over from a previous build or a previous version

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a build dependency so in order to alter the state of that GLOB you would need to manually intervene inside the CMake build's _deps folder and add/remove dlls from our fetched MPIR dependency there. I don't mind making this list manual if that makes us feel better, with that said.

foreach(mpir_dll IN LISTS chia_mpir_dlls)
add_custom_command(
TARGET ${_target}
POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${mpir_dll} $<TARGET_FILE_DIR:${_target}>
)
endforeach()
endfunction()

add_subdirectory(src)

if(EMSCRIPTEN)
Expand Down
3 changes: 2 additions & 1 deletion python-bindings/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@ FetchContent_Declare(
GIT_REPOSITORY https://github.com/pybind/pybind11.git
GIT_TAG v2.10.0
)
FetchContent_MakeAvailable(pybind11 relic)
FetchContent_MakeAvailable(pybind11)

pybind11_add_module(blspy ${CMAKE_CURRENT_SOURCE_DIR}/pythonbindings.cpp)
target_link_libraries(blspy PRIVATE bls)
handle_mpir_dlls(blspy)
178 changes: 18 additions & 160 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,9 @@
#!/usr/bin/python3
import os
import platform
import re
import subprocess
import sys
from distutils.version import LooseVersion

from setuptools import Extension, setup, setuptools
from setuptools import Extension, setup
from setuptools.command.build_ext import build_ext


Expand All @@ -19,21 +16,14 @@ def __init__(self, name, sourcedir=""):
class CMakeBuild(build_ext):
def run(self):
try:
out = subprocess.check_output(["cmake", "--version"])
subprocess.check_output(["cmake", "--version"])
except OSError:
raise RuntimeError(
"CMake must be installed to build"
+ " the following extensions: "
+ ", ".join(e.name for e in self.extensions)
)

if platform.system() == "Windows":
cmake_version = LooseVersion(
re.search(r"version\s*([\d.]+)", out.decode()).group(1)
)
if cmake_version < "3.1.0":
raise RuntimeError("CMake >= 3.1.0 is required on Windows")

for ext in self.extensions:
self.build_extension(ext)

Expand All @@ -42,6 +32,8 @@ def build_extension(self, ext):
cmake_args = [
"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=" + extdir,
"-DPYTHON_EXECUTABLE=" + sys.executable,
"-DBUILD_BLS_TESTS=0",
"-DBUILD_BLS_BENCHMARKS=0",
]

cfg = "Debug" if self.debug else "Release"
Expand All @@ -51,9 +43,6 @@ def build_extension(self, ext):
cmake_args += [
"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}".format(cfg.upper(), extdir)
]
if sys.maxsize > 2 ** 32:
cmake_args += ["-A", "x64"]
build_args += ["--", "/m"]
else:
cmake_args += ["-DCMAKE_BUILD_TYPE=" + cfg]
build_args += ["--", "-j", "6"]
Expand All @@ -72,148 +61,17 @@ def build_extension(self, ext):
)


class get_pybind_include(object):
"""Helper class to determine the pybind11 include path

The purpose of this class is to postpone importing pybind11
until it is actually installed, so that the ``get_include()``
method can be invoked."""

def __init__(self, user=False):
self.user = user

def __str__(self):
import pybind11

return pybind11.get_include(self.user)


ext_modules = [
Extension(
"blspy",
[
"src/elements.cpp",
"src/schemes.cpp",
"src/privatekey.cpp",
"src/bls.cpp",
"python-bindings/pythonbindings.cpp",
],
include_dirs=[
# Path to pybind11 headers
get_pybind_include(),
get_pybind_include(user=True),
"relic_ietf_64/include",
"mpir_gc_x64",
"libsodium/include",
],
library_dirs=[
"relic_ietf_64",
"mpir_gc_x64",
"libsodium/x64/Release/v142/static",
],
libraries=["relic_s", "Advapi32", "mpir", "libsodium"],
language="c++",
),
]


# As of Python 3.6, CCompiler has a `has_flag` method.
# cf http://bugs.python.org/issue26689
def has_flag(compiler, flagname):
"""Return a boolean indicating whether a flag name is supported on
the specified compiler.
"""
import tempfile

with tempfile.NamedTemporaryFile("w", suffix=".cpp") as f:
f.write("int main (int argc, char **argv) { return 0; }")
try:
compiler.compile([f.name], extra_postargs=[flagname])
except setuptools.distutils.errors.CompileError:
return False
return True


def cpp_flag(compiler):
"""Return the -std=c++[11/14/17] compiler flag.

The newer version is prefered over c++11 (when it is available).
"""
flags = ["-std=c++17", "-std=c++14", "-std=c++11"]

for flag in flags:
if has_flag(compiler, flag):
return flag

raise RuntimeError("Unsupported compiler -- at least C++11 support " "is needed!")


class BuildExt(build_ext):
"""A custom build extension for adding compiler-specific options."""

c_opts = {
"msvc": ["/EHsc", "/std:c++17", "/DBLSALLOC_SODIUM=1", "/DSODIUM_STATIC"],
"unix": [],
}
l_opts = {
"msvc": [],
"unix": [],
}

if sys.platform == "darwin":
darwin_opts = ["-stdlib=libc++", "-mmacosx-version-min=10.14"]
c_opts["unix"] += darwin_opts
l_opts["unix"] += darwin_opts

def build_extensions(self):
ct = self.compiler.compiler_type
opts = self.c_opts.get(ct, [])
link_opts = self.l_opts.get(ct, [])
if ct == "unix":
opts.append('-DVERSION_INFO="%s"' % self.distribution.get_version())
opts.append(cpp_flag(self.compiler))
if has_flag(self.compiler, "-fvisibility=hidden"):
opts.append("-fvisibility=hidden")
elif ct == "msvc":
if sys.version_info < (3, 9):
ver_flag = '/DVERSION_INFO=\"%s\"'
else:
ver_flag = '-DVERSION_INFO="%s"'
opts.append(ver_flag % self.distribution.get_version())
for ext in self.extensions:
ext.extra_compile_args = opts
ext.extra_link_args = link_opts
build_ext.build_extensions(self)


if platform.system() == "Windows":
setup(
name="blspy",
author="Mariano Sorgente",
author_email="[email protected]",
description="BLS signatures in c++ (with python bindings)",
long_description=open("README.md").read(),
long_description_content_type="text/markdown",
url="https://github.com/Chia-Network/bls-signatures",
python_requires=">=3.7",
setup_requires=["pybind11>=2.10.0"],
install_requires=["pybind11>=2.10.0"],
ext_modules=ext_modules,
cmdclass={"build_ext": BuildExt},
zip_safe=False,
)
else:
setup(
name="blspy",
author="Mariano Sorgente",
author_email="[email protected]",
description="BLS signatures in c++ (python bindings)",
python_requires=">=3.7",
install_requires=["wheel"],
long_description=open("README.md").read(),
long_description_content_type="text/markdown",
url="https://github.com/Chia-Network/bls-signatures",
ext_modules=[CMakeExtension("blspy", ".")],
cmdclass=dict(build_ext=CMakeBuild),
zip_safe=False,
)
setup(
name="blspy",
author="Mariano Sorgente",
author_email="[email protected]",
description="BLS signatures in c++ (python bindings)",
python_requires=">=3.7",
install_requires=["wheel"],
long_description=open("README.md").read(),
long_description_content_type="text/markdown",
url="https://github.com/Chia-Network/bls-signatures",
ext_modules=[CMakeExtension("blspy", ".")],
cmdclass=dict(build_ext=CMakeBuild),
zip_safe=False,
)
4 changes: 4 additions & 0 deletions src/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ install(DIRECTORY ${relic_BINARY_DIR}/include/ DESTINATION include/chiabls)
install(FILES ${HEADERS} DESTINATION include/chiabls)
install(FILES $<TARGET_FILE:bls> DESTINATION lib)

if(MSVC AND (BUILD_BLS_TESTS OR BUILD_BLS_BENCHMARKS))
handle_mpir_dlls(bls)
endif()

if(BUILD_BLS_TESTS)
FetchContent_Declare(
Catch2
Expand Down