improved robustness of conda installation; allowing conda packaging f…

…rom github runner for mac and linux
yumengch · yumengch · Jul 24, 2024 · Jul 19, 2024 · Jul 19, 2024 · Jul 19, 2024
commit 2cf2e22601517acfd76ba34fd3ffd4ef6a8985e6
diff --git a/.github/workflows/condaBuildMac.yaml → .github/workflows/conda_build_linux.yaml b/.github/workflows/condaBuildMac.yaml → .github/workflows/conda_build_linux.yaml
@@ -1,23 +1,31 @@
-name: condaBuild
+name: conda_build_linux
 on: [workflow_dispatch]
 jobs:
-  check_mac_installation:
-    runs-on: macos-latest
+  upload_conda_linux:
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: 'true'
-          ref: intelMacConda
+          ref: upgrade2.2.1
 
       - uses: conda-incubator/setup-miniconda@v3
         with:
           auto-activate-base: true
           auto-update-conda: true
+          activate-environment: ""
 
       - name: install pyPDAF
         shell: bash -el {0}
         run: |
+          cp PDAFBuild/setup_conda_linux.cfg setup.cfg
           conda install python anaconda-client conda-build conda-verify
           anaconda login --username yumengch --password ${{ secrets.ANACONDA }}
           conda  config --set anaconda_upload yes
-          conda-build -c conda-forge -c intel conda.recipe/
+          conda-build -c conda-forge conda.recipe/
+          conda install -y -c conda-forge --use-local pypdaf
+          mpiexec -n 4 python -u example/main.py 
+
+      - name: Setup tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3
diff --git a/.github/workflows/conda_build_mac_intel.yaml b/.github/workflows/conda_build_mac_intel.yaml
@@ -0,0 +1,37 @@
+name: conda_build_mac_intel
+on: [workflow_dispatch]
+jobs:
+  upload_conda_mac_intel:
+    runs-on: macos-13
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'true'
+          ref: upgrade2.2.1
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          auto-activate-base: true
+          auto-update-conda: true
+          activate-environment: ""
+
+      - name: build pyPDAF
+        shell: bash -el {0}
+        run: |
+          cp PDAFBuild/setup_conda_intel_mac.cfg setup.cfg
+          conda install python anaconda-client conda-build conda-verify
+          anaconda login --username yumengch --password ${{ secrets.ANACONDA }}
+          conda  config --set anaconda_upload yes
+          conda-build -c conda-forge conda.recipe/
+
+      - name: install pyPDAF
+        shell: bash -el {0}
+        run: |
+          conda install -y -c conda-forge --use-local pypdaf
+          mv example/config_mac.py example/config.py
+          mv example/config_obsA_mac.py example/config_obsA.py
+          mpiexec -n 4 python -u example/main.py 
+
+      - name: Setup tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3
diff --git a/.github/workflows/conda_build_mac_m1.yaml b/.github/workflows/conda_build_mac_m1.yaml
@@ -0,0 +1,37 @@
+name: conda_build_mac_m1
+on: [workflow_dispatch]
+jobs:
+  upload_conda_mac_m1:
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'true'
+          ref: upgrade2.2.1
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          auto-activate-base: true
+          auto-update-conda: true
+          activate-environment: ""
+
+      - name: build pyPDAF
+        shell: bash -el {0}
+        run: |
+          cp PDAFBuild/setup_conda_m1_mac.cfg setup.cfg
+          conda install python anaconda-client conda-build conda-verify
+          anaconda login --username yumengch --password ${{ secrets.ANACONDA }}
+          conda  config --set anaconda_upload yes
+          conda-build -c conda-forge conda.recipe/
+
+      - name: install pyPDAF
+        shell: bash -el {0}
+        run: |
+          conda install -y -c conda-forge --use-local pypdaf
+          mv example/config_mac.py example/config.py
+          mv example/config_obsA_mac.py example/config_obsA.py
+          mpiexec -n 4 python -u example/main.py 
+
+      - name: Setup tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3
diff --git a/.github/workflows/test_linux_build.yaml b/.github/workflows/test_linux_build.yaml
@@ -9,7 +9,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           submodules: 'true'
-          ref: upgrade2.2.1
+          ref: main
 
       - name: installing dependent libraries
         run: sudo apt install -y liblapack-dev libblas-dev libopenmpi-dev

diff --git a/.github/workflows/test_win_build.yaml b/.github/workflows/test_win_build.yaml
@@ -64,7 +64,7 @@ jobs:
       - name: install pyPDAF
         run: |
           cmd.exe /k "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat"
-          cmd.exe /k "C:\Program Files (x86)\Intel\oneAPI\compiler\latest\env\vars.bat"
+          cmd.exe /k "C:\Program Files (x86)\Intel\oneAPI\compiler\2024.2\env\vars.bat"
           cmd.exe /k '"C:\Program Files (x86)\Intel\oneAPI\mkl\latest\env\vars.bat" && powershell'
           cd D:\a\pyPDAF\pyPDAF
           Move-Item -Path setup_win.cfg -Destination setup.cfg -Force

diff --git a/PDAFBuild/linux_gfortran_openmpi_conda.cmake b/PDAFBuild/linux_gfortran_openmpi_conda.cmake
@@ -0,0 +1,22 @@
+# Set the PDAF library name, it can be pdaf-var or pdaf-d
+set(PDAF_NAME "pdaf-var")
+
+# set compiler executable
+set(CMAKE_Fortran_COMPILER "mpif90")
+# Set compiler flags for Release/Production configurations
+set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -ffree-line-length-none -fdefault-real-8 -fPIC")
+# Set compiler flags for Debug configurations
+set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -Wall -Wextra -g -pedantic -fcheck=all -fbacktrace -ffree-line-length-none -fdefault-real-8 -fPIC")
+
+# set MPI library information
+set(MPI_Fortran_INCLUDE_PATH
+    ""
+    CACHE STRING "path to the include directory of MPI_Fortran")
+set(MPI_Fortran_MODULE_DIR
+    "" 
+    CACHE STRING "path to the module directory of MPI_Fortran")
+
+# Check if the build type is not explicitly set and default to Release
+if(NOT CMAKE_BUILD_TYPE)
+  set(CMAKE_BUILD_TYPE Release)
+endif()
diff --git a/PDAFBuild/osx_gfortran_openmpi_conda.cmake b/PDAFBuild/osx_gfortran_openmpi_conda.cmake
@@ -0,0 +1,22 @@
+# Set the PDAF library name, it can be pdaf-var or pdaf-d
+set(PDAF_NAME "pdaf-var")
+
+# set compiler executable
+set(CMAKE_Fortran_COMPILER "mpif90")
+# Set compiler flags for Release/Production configurations
+set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -ffree-line-length-none -fdefault-real-8 -fPIC  -mmacosx-version-min=11.0")
+# Set compiler flags for Debug configurations
+set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -Wall -Wextra -g -pedantic -fcheck=all -fbacktrace -ffree-line-length-none -fdefault-real-8 -fPIC")
+
+# set MPI library information
+set(MPI_Fortran_INCLUDE_PATH
+    ""
+    CACHE STRING "path to the include directory of MPI_Fortran")
+set(MPI_Fortran_MODULE_DIR
+    "" 
+    CACHE STRING "path to the module directory of MPI_Fortran")
+
+# Check if the build type is not explicitly set and default to Release
+if(NOT CMAKE_BUILD_TYPE)
+  set(CMAKE_BUILD_TYPE Release)
+endif()
diff --git a/PDAFBuild/setup_conda_intel_mac.cfg b/PDAFBuild/setup_conda_intel_mac.cfg
@@ -0,0 +1,23 @@
+[pyPDAF]
+# working directory of pip
+pwd = /Users/runner/work/pyPDAF/pyPDAF/
+# PDAF source file directory
+PDAF_dir = PDAF_V2.2.1
+cmake_config_path = /Users/runner/work/pyPDAF/pyPDAF/PDAFBuild/osx_gfortran_openmpi_conda.cmake
+# switch if the package is built with conda
+condaBuild = True
+# compilers used in fortran and C
+# c_compiler can be: gcc, msvc, icc, clang
+# fortran compiler can be: gfortran, ifort
+# fortran compiler should be consistent with the compiler used in cmake configuration file
+c_compiler = clang
+fortran_compiler = gfortran
+# if MKL is used, give the path to the static MKL library
+use_MKL=False
+MKLROOT=
+# if dynamic/shared liblapack and libblas library is used,
+# give the library path and flags
+LAPACK_PATH=
+LAPACK_Flag=lapack,blas
+# GIVE MPI information
+MPI_LIB_PATH=
diff --git a/PDAFBuild/setup_conda_linux.cfg b/PDAFBuild/setup_conda_linux.cfg
@@ -0,0 +1,20 @@
+[pyPDAF]
+pwd = /home/runner/work/pyPDAF/pyPDAF
+PDAF_dir = PDAF_V2.2.1
+cmake_config_path = /home/runner/work/pyPDAF/pyPDAF/PDAFBuild/linux_gfortran_openmpi_conda.cmake
+condaBuild = True
+# compilers used in fortran and C
+# c_compiler can be: gcc, msvc, icc, clang
+# fortran compiler can be: gfortran, ifort
+# fortran compiler should be consistent with the compiler used in cmake configuration file
+c_compiler = gcc
+fortran_compiler = gfortran
+# if MKL is used, give the path to the static MKL library
+use_MKL= True
+MKLROOT=
+# if dynamic/shared liblapack and libblas library is used,
+# give the library path and flags
+LAPACK_PATH=
+LAPACK_Flag=lapack,blas
+# GIVE MPI information
+MPI_LIB_PATH=
diff --git a/PDAFBuild/setup_conda_m1_mac.cfg b/PDAFBuild/setup_conda_m1_mac.cfg
@@ -0,0 +1,23 @@
+[pyPDAF]
+# working directory of pip
+pwd = /Users/runner/work/pyPDAF/pyPDAF/
+# PDAF source file directory
+PDAF_dir = PDAF_V2.2.1
+cmake_config_path = /Users/runner/work/pyPDAF/pyPDAF/PDAFBuild/osx_gfortran_openmpi_conda.cmake
+# switch if the package is built with conda
+condaBuild = True
+# compilers used in fortran and C
+# c_compiler can be: gcc, msvc, icc, clang
+# fortran compiler can be: gfortran, ifort
+# fortran compiler should be consistent with the compiler used in cmake configuration file
+c_compiler = clang
+fortran_compiler = gfortran
+# if MKL is used, give the path to the static MKL library
+use_MKL=False
+MKLROOT=
+# if dynamic/shared liblapack and libblas library is used,
+# give the library path and flags
+LAPACK_PATH=
+LAPACK_Flag=lapack,blas
+# GIVE MPI information
+MPI_LIB_PATH=
diff --git a/setup_win.cfg → PDAFBuild/setup_conda_win.cfg b/setup_win.cfg → PDAFBuild/setup_conda_win.cfg
@@ -1,19 +1,17 @@
-[options]
-include_package_data = True
-install_requires =
-    Cython
-    mpi4py
-    numpy
-
 [pyPDAF]
 pwd = D:\a\pyPDAF\pyPDAF
 PDAF_dir = PDAF_V2.2.1
 cmake_config_path = D:\a\pyPDAF\pyPDAF\PDAFBuild\win_intel_msmpi_pypdaf.cmake
-CC = cl
-condaBuild = False
+condaBuild = True
+# compilers used in fortran and C
+# c_compiler can be: gcc, msvc, icc, clang
+# fortran compiler can be: gfortran, ifort
+# fortran compiler should be consistent with the compiler used in cmake configuration file
+c_compiler = msvc
+fortran_compiler = ifort
 # if MKL is used, give the path to the static MKL library
 use_MKL= True
-MKLROOT=C:\Program Files (x86)\Intel\oneAPI\mkl\2024.2\lib
+MKLROOT=
 # if dynamic/shared liblapack and libblas library is used,
 # give the library path and flags
 LAPACK_PATH=

diff --git a/PDAFBuild/setup_mac.cfg b/PDAFBuild/setup_mac.cfg
@@ -0,0 +1,20 @@
+[pyPDAF]
+pwd = /Users/runner/work/pyPDAF/pyPDAF/
+PDAF_dir = PDAF_V2.2.1
+cmake_config_path = /Users/runner/work/pyPDAF/pyPDAF/PDAFBuild/osx_gfortran_openmpi_pypdaf.cmake
+condaBuild = False
+# compilers used in fortran and C
+# c_compiler can be: gcc, msvc, icc, clang
+# fortran compiler can be: gfortran, ifort
+# fortran compiler should be consistent with the compiler used in cmake configuration file
+c_compiler = clang
+fortran_compiler = gfortran
+# if MKL is used, give the path to the static MKL library
+use_MKL= True
+MKLROOT=
+# if dynamic/shared liblapack and libblas library is used,
+# give the library path and flags
+LAPACK_PATH=
+LAPACK_Flag=lapack,blas
+# GIVE MPI information
+MPI_LIB_PATH=
diff --git a/PDAFBuild/setup_win.cfg b/PDAFBuild/setup_win.cfg
@@ -0,0 +1,21 @@
+[pyPDAF]
+pwd = D:\a\pyPDAF\pyPDAF
+PDAF_dir = PDAF_V2.2.1
+cmake_config_path = D:\a\pyPDAF\pyPDAF\PDAFBuild\win_intel_msmpi_pypdaf.cmake
+condaBuild = False
+# compilers used in fortran and C
+# c_compiler can be: gcc, msvc, icc, clang
+# fortran compiler can be: gfortran, ifort
+# fortran compiler should be consistent with the compiler used in cmake configuration file
+c_compiler = msvc
+fortran_compiler = ifort
+# if MKL is used, give the path to the static MKL library
+use_MKL= True
+MKLROOT= C:\Program Files (x86)\Intel\oneAPI\mkl\latest\lib
+# if dynamic/shared liblapack and libblas library is used,
+# give the library path and flags
+LAPACK_PATH=
+LAPACK_Flag=lapack,blas
+# GIVE MPI information
+MPI_LIB_PATH=C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64
+
diff --git a/conda.recipe/bld.bat b/conda.recipe/bld.bat
@@ -1,3 +1,3 @@
 call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" intel64 vs2022
 
-%PYTHON -m pip install . --no-deps --ignore-installed --no-cache-dir -vvv
+"%PYTHON%" -m pip install . --no-deps --ignore-installed --no-cache-dir -vvv
diff --git a/conda.recipe/build.sh b/conda.recipe/build.sh
@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 set -ex
 
-
 # Install the Python package, but without dependencies,
 # because Conda takes care of that
 $PYTHON -m pip install . --no-deps --ignore-installed --no-cache-dir -vvv
diff --git a/conda.recipe/conda_build_config.yaml b/conda.recipe/conda_build_config.yaml
@@ -1,8 +1,8 @@
 python:
-    - 3.8      # [not osx]
     - 3.9
     - 3.10
     - 3.11
+    - 3.12 # [not win]
 
 c_compiler:
     - vs2022   # [win]

diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml
@@ -1,8 +1,8 @@
-{% set version = "0.02" %}
+{% set version = "0.03" %}
 
 package:
   name: pypdaf
-  version: 0.0.2
+  version: 0.0.3
 
 source:
   noarch: python
@@ -14,25 +14,28 @@ build:
 requirements:
   host:
     - python {{python}}
-    - cython
-    - pip
     - setuptools
+    - pip
+    - cython
     - numpy
     - mpi4py
     - {{ mpi }}
-    - mkl-static                 # [x86]
-    - blas-devel                 # [not x86]
-    - liblapack                  # [not x86]
+    - mkl-static                 # [not osx]
+    - blas-devel=*=*netlib*      # [osx]
+    - liblapack=*=*netlib*       # [osx]
   build:
     - {{ compiler('c') }}
     - {{ compiler('fortran') }}  # [not win]
-    - cmake
+    - cmake   # [not win]
+    - make    # [not win]
   run:
     - python
     - {{ pin_compatible('numpy') }}
     - mpi4py
-    - intel-fortran-rt # [win]
+    - intel-fortran-rt=2024.2.0 # [win]
     - {{ mpi }}
+    - blas-devel=*=*netlib* # [osx]
+    - liblapack=*=*netlib* # [osx]
 
 about:
   home: https://github.com/yumengch/pyPDAF

diff --git a/pyPDAF/__init__.py b/pyPDAF/__init__.py