testing the conda installation for linux and mac

testing conda build in github runner with linux and osx testing conda build installation for easier workflow correct cfg path
yumengch · Jul 23, 2024 · e109d5d · e109d5d
1 parent ef11b9d
commit e109d5d
Show file tree

Hide file tree

Showing 20 changed files with 283 additions and 175 deletions.
diff --git a/.github/workflows/condaBuildMac.yaml → .github/workflows/conda_build_linux.yaml b/.github/workflows/condaBuildMac.yaml → .github/workflows/conda_build_linux.yaml
@@ -1,23 +1,31 @@
-name: condaBuild
+name: conda_build_linux
 on: [workflow_dispatch]
 jobs:
-  check_mac_installation:
-    runs-on: macos-latest
+  upload_conda_linux:
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: 'true'
-          ref: intelMacConda
+          ref: upgrade2.2.1
 
       - uses: conda-incubator/setup-miniconda@v3
         with:
           auto-activate-base: true
           auto-update-conda: true
+          activate-environment: ""
 
       - name: install pyPDAF
         shell: bash -el {0}
         run: |
+          cp PDAFBuild/setup_conda_linux.cfg setup.cfg
           conda install python anaconda-client conda-build conda-verify
           anaconda login --username yumengch --password ${{ secrets.ANACONDA }}
           conda  config --set anaconda_upload yes
-          conda-build -c conda-forge -c intel conda.recipe/
+          conda-build -c conda-forge conda.recipe/
+          conda install -y -c conda-forge --use-local pypdaf
+          mpiexec -n 4 python -u example/main.py 
+
+      - name: Setup tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3
diff --git a/.github/workflows/conda_build_mac_intel.yaml b/.github/workflows/conda_build_mac_intel.yaml
@@ -0,0 +1,37 @@
+name: conda_build_mac_intel
+on: [workflow_dispatch, pull_request]
+jobs:
+  upload_conda_mac_intel:
+    runs-on: macos-13
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'true'
+          ref: upgrade2.2.1
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          auto-activate-base: true
+          auto-update-conda: true
+          activate-environment: ""
+
+      - name: build pyPDAF
+        shell: bash -el {0}
+        run: |
+          cp PDAFBuild/setup_conda_intel_mac.cfg setup.cfg
+          conda install python anaconda-client conda-build conda-verify
+          anaconda login --username yumengch --password ${{ secrets.ANACONDA }}
+          conda  config --set anaconda_upload no
+          conda-build -c conda-forge conda.recipe/
+
+      - name: install pyPDAF
+        shell: bash -el {0}
+        run: |
+          conda install -y -c conda-forge --use-local pypdaf
+          mv example/config_mac.py example/config.py
+          mv example/config_obsA_mac.py example/config_obsA.py
+          mpiexec -n 4 python -u example/main.py 
+
+      - name: Setup tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3
diff --git a/.github/workflows/conda_build_mac_m1.yaml b/.github/workflows/conda_build_mac_m1.yaml
@@ -0,0 +1,37 @@
+name: conda_build_mac_m1
+on: [workflow_dispatch, pull_request]
+jobs:
+  upload_conda_mac_m1:
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'true'
+          ref: upgrade2.2.1
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          auto-activate-base: true
+          auto-update-conda: true
+          activate-environment: ""
+
+      - name: build pyPDAF
+        shell: bash -el {0}
+        run: |
+          cp PDAFBuild/setup_conda_m1_mac.cfg setup.cfg
+          conda install python anaconda-client conda-build conda-verify
+          anaconda login --username yumengch --password ${{ secrets.ANACONDA }}
+          conda  config --set anaconda_upload no
+          conda-build -c conda-forge conda.recipe/
+
+      - name: install pyPDAF
+        shell: bash -el {0}
+        run: |
+          conda install -y -c conda-forge --use-local pypdaf
+          mv example/config_mac.py example/config.py
+          mv example/config_obsA_mac.py example/config_obsA.py
+          mpiexec -n 4 python -u example/main.py 
+
+      - name: Setup tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3
diff --git a/.github/workflows/test_linux_build.yaml b/.github/workflows/test_linux_build.yaml
@@ -9,7 +9,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           submodules: 'true'
-          ref: upgrade2.2.1
+          ref: main
 
       - name: installing dependent libraries
         run: sudo apt install -y liblapack-dev libblas-dev libopenmpi-dev

diff --git a/PDAFBuild/linux_gfortran_openmpi_conda.cmake b/PDAFBuild/linux_gfortran_openmpi_conda.cmake
@@ -0,0 +1,22 @@
+# Set the PDAF library name, it can be pdaf-var or pdaf-d
+set(PDAF_NAME "pdaf-var")
+
+# set compiler executable
+set(CMAKE_Fortran_COMPILER "mpif90")
+# Set compiler flags for Release/Production configurations
+set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -ffree-line-length-none -fdefault-real-8 -fPIC")
+# Set compiler flags for Debug configurations
+set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -Wall -Wextra -g -pedantic -fcheck=all -fbacktrace -ffree-line-length-none -fdefault-real-8 -fPIC")
+
+# set MPI library information
+set(MPI_Fortran_INCLUDE_PATH
+    ""
+    CACHE STRING "path to the include directory of MPI_Fortran")
+set(MPI_Fortran_MODULE_DIR
+    "" 
+    CACHE STRING "path to the module directory of MPI_Fortran")
+
+# Check if the build type is not explicitly set and default to Release
+if(NOT CMAKE_BUILD_TYPE)
+  set(CMAKE_BUILD_TYPE Release)
+endif()
diff --git a/PDAFBuild/osx_gfortran_openmpi_conda.cmake b/PDAFBuild/osx_gfortran_openmpi_conda.cmake
@@ -0,0 +1,22 @@
+# Set the PDAF library name, it can be pdaf-var or pdaf-d
+set(PDAF_NAME "pdaf-var")
+
+# set compiler executable
+set(CMAKE_Fortran_COMPILER "mpif90")
+# Set compiler flags for Release/Production configurations
+set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -ffree-line-length-none -fdefault-real-8 -fPIC  -mmacosx-version-min=11.0")
+# Set compiler flags for Debug configurations
+set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -Wall -Wextra -g -pedantic -fcheck=all -fbacktrace -ffree-line-length-none -fdefault-real-8 -fPIC")
+
+# set MPI library information
+set(MPI_Fortran_INCLUDE_PATH
+    ""
+    CACHE STRING "path to the include directory of MPI_Fortran")
+set(MPI_Fortran_MODULE_DIR
+    "" 
+    CACHE STRING "path to the module directory of MPI_Fortran")
+
+# Check if the build type is not explicitly set and default to Release
+if(NOT CMAKE_BUILD_TYPE)
+  set(CMAKE_BUILD_TYPE Release)
+endif()
diff --git a/PDAFBuild/setup_conda_intel_mac.cfg b/PDAFBuild/setup_conda_intel_mac.cfg
@@ -0,0 +1,23 @@
+[pyPDAF]
+# working directory of pip
+pwd = /Users/runner/work/pyPDAF/pyPDAF/
+# PDAF source file directory
+PDAF_dir = PDAF_V2.2.1
+cmake_config_path = /Users/runner/work/pyPDAF/pyPDAF/PDAFBuild/osx_gfortran_openmpi_conda.cmake
+# switch if the package is built with conda
+condaBuild = True
+# compilers used in fortran and C
+# c_compiler can be: gcc, msvc, icc, clang
+# fortran compiler can be: gfortran, ifort
+# fortran compiler should be consistent with the compiler used in cmake configuration file
+c_compiler = clang
+fortran_compiler = gfortran
+# if MKL is used, give the path to the static MKL library
+use_MKL=False
+MKLROOT=
+# if dynamic/shared liblapack and libblas library is used,
+# give the library path and flags
+LAPACK_PATH=
+LAPACK_Flag=lapack,blas
+# GIVE MPI information
+MPI_LIB_PATH=
diff --git a/PDAFBuild/setup_conda_linux.cfg b/PDAFBuild/setup_conda_linux.cfg
@@ -0,0 +1,20 @@
+[pyPDAF]
+pwd = /home/runner/work/pyPDAF/pyPDAF
+PDAF_dir = PDAF_V2.2.1
+cmake_config_path = /home/runner/work/pyPDAF/pyPDAF/PDAFBuild/linux_gfortran_openmpi_conda.cmake
+condaBuild = True
+# compilers used in fortran and C
+# c_compiler can be: gcc, msvc, icc, clang
+# fortran compiler can be: gfortran, ifort
+# fortran compiler should be consistent with the compiler used in cmake configuration file
+c_compiler = gcc
+fortran_compiler = gfortran
+# if MKL is used, give the path to the static MKL library
+use_MKL= True
+MKLROOT=
+# if dynamic/shared liblapack and libblas library is used,
+# give the library path and flags
+LAPACK_PATH=
+LAPACK_Flag=lapack,blas
+# GIVE MPI information
+MPI_LIB_PATH=
diff --git a/PDAFBuild/setup_conda_m1_mac.cfg b/PDAFBuild/setup_conda_m1_mac.cfg
@@ -0,0 +1,23 @@
+[pyPDAF]
+# working directory of pip
+pwd = /Users/runner/work/pyPDAF/pyPDAF/
+# PDAF source file directory
+PDAF_dir = PDAF_V2.2.1
+cmake_config_path = /Users/runner/work/pyPDAF/pyPDAF/PDAFBuild/osx_gfortran_openmpi_conda.cmake
+# switch if the package is built with conda
+condaBuild = True
+# compilers used in fortran and C
+# c_compiler can be: gcc, msvc, icc, clang
+# fortran compiler can be: gfortran, ifort
+# fortran compiler should be consistent with the compiler used in cmake configuration file
+c_compiler = clang
+fortran_compiler = gfortran
+# if MKL is used, give the path to the static MKL library
+use_MKL=False
+MKLROOT=
+# if dynamic/shared liblapack and libblas library is used,
+# give the library path and flags
+LAPACK_PATH=
+LAPACK_Flag=lapack,blas
+# GIVE MPI information
+MPI_LIB_PATH=
diff --git a/PDAFBuild/setup_mac.cfg b/PDAFBuild/setup_mac.cfg
@@ -0,0 +1,20 @@
+[pyPDAF]
+pwd = /Users/runner/work/pyPDAF/pyPDAF/
+PDAF_dir = PDAF_V2.2.1
+cmake_config_path = /Users/runner/work/pyPDAF/pyPDAF/PDAFBuild/osx_gfortran_openmpi_pypdaf.cmake
+condaBuild = False
+# compilers used in fortran and C
+# c_compiler can be: gcc, msvc, icc, clang
+# fortran compiler can be: gfortran, ifort
+# fortran compiler should be consistent with the compiler used in cmake configuration file
+c_compiler = clang
+fortran_compiler = gfortran
+# if MKL is used, give the path to the static MKL library
+use_MKL= True
+MKLROOT=
+# if dynamic/shared liblapack and libblas library is used,
+# give the library path and flags
+LAPACK_PATH=
+LAPACK_Flag=lapack,blas
+# GIVE MPI information
+MPI_LIB_PATH=
diff --git a/setup_win.cfg → PDAFBuild/setup_win.cfg b/setup_win.cfg → PDAFBuild/setup_win.cfg
@@ -1,22 +1,21 @@
-[options]
-include_package_data = True
-install_requires =
-    Cython
-    mpi4py
-    numpy
-
 [pyPDAF]
 pwd = D:\a\pyPDAF\pyPDAF
 PDAF_dir = PDAF_V2.2.1
 cmake_config_path = D:\a\pyPDAF\pyPDAF\PDAFBuild\win_intel_msmpi_pypdaf.cmake
-CC = cl
 condaBuild = False
+# compilers used in fortran and C
+# c_compiler can be: gcc, msvc, icc, clang
+# fortran compiler can be: gfortran, ifort
+# fortran compiler should be consistent with the compiler used in cmake configuration file
+c_compiler = msvc
+fortran_compiler = ifort
 # if MKL is used, give the path to the static MKL library
 use_MKL= True
-MKLROOT=C:\Program Files (x86)\Intel\oneAPI\mkl\2024.2\lib
+MKLROOT= C:\Program Files (x86)\Intel\oneAPI\mkl\latest\lib
 # if dynamic/shared liblapack and libblas library is used,
 # give the library path and flags
 LAPACK_PATH=
 LAPACK_Flag=lapack,blas
 # GIVE MPI information
 MPI_LIB_PATH=C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64
+
diff --git a/conda.recipe/build.sh b/conda.recipe/build.sh
@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 set -ex
 
-
 # Install the Python package, but without dependencies,
 # because Conda takes care of that
 $PYTHON -m pip install . --no-deps --ignore-installed --no-cache-dir -vvv
diff --git a/conda.recipe/conda_build_config.yaml b/conda.recipe/conda_build_config.yaml
@@ -1,8 +1,7 @@
 python:
-    - 3.8      # [not osx]
-    - 3.9
-    - 3.10
-    - 3.11
+    # - 3.10
+    # - 3.11
+    - 3.12
 
 c_compiler:
     - vs2022   # [win]

diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml
@@ -14,19 +14,20 @@ build:
 requirements:
   host:
     - python {{python}}
-    - cython
     - pip
-    - setuptools
+    - cython
     - numpy
     - mpi4py
     - {{ mpi }}
-    - mkl-static                 # [x86]
-    - blas-devel                 # [not x86]
-    - liblapack                  # [not x86]
+    - mkl-static                 # [not osx]
+    - blas-devel                 # [osx]
+    - liblapack                  # [osx]
   build:
     - {{ compiler('c') }}
     - {{ compiler('fortran') }}  # [not win]
     - cmake
+    - setuptools
+    - make # [not win]
   run:
     - python
     - {{ pin_compatible('numpy') }}

diff --git a/pyPDAF/__init__.py b/pyPDAF/__init__.py
diff --git a/pyproject.toml b/pyproject.toml
@@ -6,7 +6,7 @@ name = "pyPDAF"
 authors = [
     {name = "Yumeng Chen", email = "[email protected]"},
 ]
-version = "0.0.2"
+version = "0.0.3"
 description = "A Python interface to PDAF"
 readme = "README.md"
 requires-python = ">=3.8"

diff --git a/setup.cfg b/setup.cfg
@@ -1,29 +1,16 @@
-[metadata]
-name = pyPDAF
-version = 0.0.2
-description = A Python interface to Fortran-written PDAF data assimilation library
-long_description = file: README.md, LICENSE
-keywords = data assimilation, PDAF
-license = GPL V3
-classifiers =
-    License :: GPL V3
-    Programming Language :: Python :: 3
-
-[options]
-include_package_data = True
-install_requires =
-    Cython
-    mpi4py
-    numpy
-
 [pyPDAF]
 pwd = /home/runner/work/pyPDAF/pyPDAF
 PDAF_dir = PDAF_V2.2.1
 cmake_config_path = /home/runner/work/pyPDAF/pyPDAF/PDAFBuild/linux_gfortran_openmpi_pypdaf.cmake
-CC = mpicc
 condaBuild = False
+# compilers used in fortran and C
+# c_compiler can be: gcc, msvc, icc, clang
+# fortran compiler can be: gfortran, ifort
+# fortran compiler should be consistent with the compiler used in cmake configuration file
+c_compiler = gcc
+fortran_compiler = gfortran
 # if MKL is used, give the path to the static MKL library
-use_MKL= False
+use_MKL= True
 MKLROOT=
 # if dynamic/shared liblapack and libblas library is used,
 # give the library path and flags