Default to dynamically linking libsodium, static link on wheels (Chia…

…-Network#145) * Update win wheels, static only on CI * Catch up ARM64 CI, specify static everywhere. Verbose * Handle existing link, force sodium? * only libsodium static when $CIBUILDWHEEL - lower verbosity * Ubuntu valgrind should use static libsodium * Should be SEED=UDEV but defaulted to UDEV so past ok * Create python 3.9 wheels * temp fix Co-authored-by: Bill Blanke <[email protected]>
sarvex · Nov 16, 2020 · 4bed72b · 4bed72b
1 parent ee71adc
commit 4bed72b
Show file tree

Hide file tree

Showing 6 changed files with 40 additions and 27 deletions.
diff --git a/.github/workflows/build-aarch64.yml b/.github/workflows/build-aarch64.yml
@@ -13,7 +13,7 @@ jobs:
 
     steps:
     - name: Cancel previous runs on the same branch
-      uses: styfle/cancel-workflow-action@0.5.0
+      uses: styfle/cancel-workflow-action@0.6.0
       with:
         access_token: ${{ github.token }}
 
@@ -41,7 +41,7 @@ jobs:
         source venv/bin/activate
         python -m pip install --upgrade pip
         pip install wheel
-        python -m pip install cibuildwheel==1.6.1
+        python -m pip install cibuildwheel==1.6.4
 
     - name: Lint source with flake8
       run: |
@@ -61,14 +61,15 @@ jobs:
         python -m cibuildwheel --output-dir dist
       env:
         # build python 3.7 and 3.8
-        CIBW_BUILD: cp37-* cp38-*
+        CIBW_BUILD: cp37-* cp38-* cp39-*
         CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014
         CIBW_BUILD_VERBOSITY_LINUX: 0
         CIBW_BEFORE_BUILD_LINUX: >
           python -m pip install --upgrade pip
         CIBW_BEFORE_ALL_LINUX: >
           yum -y install epel-release
           && yum -y install cmake3 lzip
+          && rm -rf /usr/local/bin/cmake
           && ln -s /usr/bin/cmake3 /usr/local/bin/cmake
           && cmake --version
           && curl -L https://gmplib.org/download/gmp/gmp-6.1.2.tar.lz | lzip -dc | tar x
@@ -91,23 +92,23 @@ jobs:
         source venv/bin/activate
         pip install twine
 
-    - name: Publish distribution to Test PyPI
-      if: steps.check_secrets.outputs.HAS_SECRET
+    - name: Publish distribution to PyPI
+      if: startsWith(github.event.ref, 'refs/tags') && steps.check_secrets.outputs.HAS_SECRET
       env:
-        TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/
         TWINE_USERNAME: __token__
         TWINE_NON_INTERACTIVE: 1
-        TWINE_PASSWORD: ${{ secrets.test_pypi_password }}
+        TWINE_PASSWORD: ${{ secrets.pypi_password }}
       run: |
         source venv/bin/activate
         twine upload --non-interactive --skip-existing --verbose 'dist/*'
 
-    - name: Publish distribution to PyPI
-      if: startsWith(github.event.ref, 'refs/tags') && steps.check_secrets.outputs.HAS_SECRET
+    - name: Publish distribution to Test PyPI
+      if: steps.check_secrets.outputs.HAS_SECRET
       env:
+        TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/
         TWINE_USERNAME: __token__
         TWINE_NON_INTERACTIVE: 1
-        TWINE_PASSWORD: ${{ secrets.pypi_password }}
+        TWINE_PASSWORD: ${{ secrets.test_pypi_password }}
       run: |
         source venv/bin/activate
         twine upload --non-interactive --skip-existing --verbose 'dist/*'

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -9,11 +9,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-latest, ubuntu-18.04, windows-latest]
+        os: [macos-latest, ubuntu-latest, windows-latest]
 
     steps:
     - name: Cancel previous runs on the same branch
-      uses: styfle/cancel-workflow-action@0.5.0
+      uses: styfle/cancel-workflow-action@0.6.0
       with:
         access_token: ${{ github.token }}
 
@@ -41,7 +41,7 @@ jobs:
     - name: Install cibuildwheel
       run: |
        python -m pip install --upgrade pip
-       python -m pip install cibuildwheel==1.6.1
+       python -m pip install cibuildwheel==1.6.4
 
     - name: Lint source with flake8
       run: |
@@ -56,8 +56,8 @@ jobs:
     - name: Build source distribution with MacOS
       if: startsWith(matrix.os, 'mac')
       run: |
-        pip install pep517
-        python -m pep517.build --source --out-dir dist .
+        pip install build
+        python -m build --sdist --outdir dist .
 
     - name: Build wheel and test
       run: |
@@ -67,7 +67,7 @@ jobs:
         CIBW_BUILD_VERBOSITY_LINUX: 0
         CIBW_BUILD_VERBOSITY_WINDOWS: 0
         # Python 3.7 and 3.8
-        CIBW_BUILD: cp37-* cp38-*
+        CIBW_BUILD: cp37-* cp38-* cp39-*
         CIBW_SKIP: '*-manylinux_i686 *-win32'
         CIBW_TEST_REQUIRES: pytest
         CIBW_MANYLINUX_X86_64_IMAGE: manylinux2010
@@ -89,7 +89,7 @@ jobs:
           && make && make install && cd .. && rm -rf libsodium-stable
         CIBW_BEFORE_BUID_LINUX: >
           python -m pip install --upgrade pip
-        CIBW_ENVIRONMENT_MACOS: MACOSX_DEPLOYMENT_TARGET=10.14
+        CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=10.14"
         # GH hosted MacOS runner has gmp and libsodium natively
         CIBW_BEFORE_BUILD_MACOS: >
           python -m pip install --upgrade pip
@@ -99,7 +99,6 @@ jobs:
           && delocate-listdeps {wheel}
           && delocate-wheel -v {wheel}
           && cp {wheel} {dest_dir}
-        CIBW_ENVIRONMENT_WINDOWS: "BLSALLOC_SODIUM=1 SETUPTOOLS_USE_DISTUTILS=stdlib"
         CIBW_BEFORE_ALL_WINDOWS: >
           curl -L https://download.libsodium.org/libsodium/releases/libsodium-1.0.18-stable-msvc.zip > libsodium-1.0.18-stable-msvc.zip
           && 7z x libsodium-1.0.18-stable-msvc.zip
@@ -120,6 +119,7 @@ jobs:
           && delocate-wheel -v -i mpir_gc_x64/mpir_sandybridge.dll {wheel}
           && delocate-wheel -v -i mpir_gc_x64/mpir_skylake_avx.dll {wheel}
           && cp {wheel} {dest_dir} && ls -l {dest_dir}
+        # CIBW_ENVIRONMENT_WINDOWS: ""
         # The pure python implementation, is not used anywhere and will be replaced
         # pytest {project}/python-impl/tests.py
         CIBW_TEST_COMMAND: pytest -v {project}/python-bindings/test.py
@@ -149,13 +149,15 @@ jobs:
         cmake --version
         echo "attempting to curl"
         curl -L https://download.libsodium.org/libsodium/releases/libsodium-1.0.18-stable.tar.gz | tar xz
-        echo "curl complete?"
+        echo "curl complete."
         cd libsodium-stable
         ls -l
         ./configure --with-pic="yes"
         make
         sudo make install
         cd ..
+        echo "Setting libsodium to static compile."
+        export CIBUILDWHEEL=1
         mkdir -p build
         cd build
         cmake ../

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -14,13 +14,17 @@ set(CMAKE_MODULE_PATH
 	${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules
 )
 
-set(sodium_USE_STATIC_LIBS TRUE)
+# Make sure to statically link libsodium when building wheels on ci
+if(DEFINED ENV{CIBUILDWHEEL})
+  set(sodium_USE_STATIC_LIBS TRUE)
+  message("libsodium will be statically linked because $CIBUILDWHEEL is defined.")
+endif()
+
 find_package(sodium)
 if (SODIUM_FOUND)
   message(STATUS "Found libsodium")
   message(STATUS "Sodium include dir = ${sodium_INCLUDE_DIR}")
   set(BLSALLOC_SODIUM "1" CACHE STRING "")
-  set(SODIUM_STATIC "1" CACHE STRING "")
   include_directories(${sodium_INCLUDE_DIR})
 endif()
 
@@ -54,13 +58,13 @@ set(DOCUM "off" CACHE STRING "")
 set(FP_PRIME 381 CACHE INTEGER "")
 
 IF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
-  set(DSEED "UDEV" CACHE STRING "")
+  set(SEED "UDEV" CACHE STRING "")
   set(FP_QNRES "off" CACHE STRING "")
 ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
   set(SEED "WCGR" CACHE STRING "")
   set(FP_QNRES "on" CACHE STRING "")
 ELSE()
-  set(DSEED "UDEV" CACHE STRING "")
+  set(SEED "UDEV" CACHE STRING "")
   set(FP_QNRES "on" CACHE STRING "")
 ENDIF()
 set(STBIN "OFF" CACHE STRING "")

diff --git a/README.md b/README.md
@@ -208,7 +208,9 @@ Libsodium and GMP are optional dependencies: libsodium gives secure memory
 allocation, and GMP speeds up the library by ~ 3x. To install them, either
 download them from github and follow the instructions for each repo, or use
 a package manager like APT or brew. You can follow the recipe used to build
-python wheels for multiple platforms in `.github/workflows/`
+python wheels for multiple platforms in `.github/workflows/`. libsodium is
+dynamically linked unless the environment variable $CIBUILDWHEEL is set which
+will then cause libsodium to statically link.
 
 ## Discussion
 
@@ -234,10 +236,10 @@ build binary wheels for MacOS, Linux (x64 and aarch64), and Windows and publish
 them with a source wheel on PyPi. See `.github/workflows/build.yml`. CMake uses
 [FetchContent](https://cmake.org/cmake/help/latest/module/FetchContent.html)
 to download [pybind11](https://github.com/pybind/pybind11) for the Python
-bindings and relic from a chia relic forked reporitory. Building is then
+bindings and relic from a chia relic forked repository. Building is then
 managed by [cibuildwheel](https://github.com/joerick/cibuildwheel).
 Further installation is then available via `pip install blspy` e.g. The ci
-builds include GMP and libsoduium.
+builds include GMP and a statically linked libsodium.
 
 ## Contributing and workflow
 

diff --git a/python-bindings/pythonbindings.cpp b/python-bindings/pythonbindings.cpp
@@ -618,7 +618,11 @@ PYBIND11_MODULE(blspy, m)
     m.attr("SignatureMPL") = m.attr("G2Element");
 
 #ifdef VERSION_INFO
+#ifdef _WIN32
+    m.attr("__version__") = "dev";
+#else
     m.attr("__version__") = VERSION_INFO;
+#endif
 #else
     m.attr("__version__") = "dev";
 #endif

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -33,7 +33,7 @@ add_library(blstmp ${HEADERS}
   ${CMAKE_CURRENT_SOURCE_DIR}/schemes.cpp
 )
 
-target_compile_definitions(blstmp PRIVATE BLSALLOC_SODIUM=1 SODIUM_STATIC=1)
+target_compile_definitions(blstmp PRIVATE BLSALLOC_SODIUM=1)
 
 set(OPREFIX object_)
 find_library(GMP_NAME NAMES libgmp.a gmp)