Adding cpu ci for conda build (#482)

Authors: - Corey J. Nolet (https://github.com/cjnolet) Approvers: - Mark Sadang (https://github.com/msadang) URL: #482
rapidsai · Feb 2, 2022 · 207a9de · 207a9de
1 parent fc4c210
commit 207a9de
Show file tree

Hide file tree

Showing 3 changed files with 211 additions and 0 deletions.
diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh
@@ -0,0 +1,121 @@
+#!/bin/bash
+# Copyright (c) 2022, NVIDIA CORPORATION.
+#########################################
+#  RAFT CPU conda build script for CI   #
+#########################################
+set -e
+
+# Set path and build parallel level
+# openmpi dir is required on CentOS for finding MPI libs from cmake
+if [[ -e /etc/os-release ]] && (grep -qi centos /etc/os-release); then
+    export PATH=/opt/conda/bin:/usr/local/cuda/bin:/usr/lib64/openmpi/bin:$PATH
+else
+    export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
+fi
+export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}
+
+# Set home to the job's workspace
+export HOME=$WORKSPACE
+
+# Switch to project root; also root of repo checkout
+cd $WORKSPACE
+
+# If nightly build, append current YYMMDD to version
+if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then
+  export VERSION_SUFFIX=`date +%y%m%d`
+fi
+
+# Setup 'gpuci_conda_retry' for build retries (results in 2 total attempts)
+export GPUCI_CONDA_RETRY_MAX=1
+export GPUCI_CONDA_RETRY_SLEEP=30
+
+# Use Ninja to build
+export CMAKE_GENERATOR="Ninja"
+export CONDA_BLD_DIR="${WORKSPACE}/.conda-bld"
+
+# ucx-py version
+export UCX_PY_VERSION='0.25.*'
+
+################################################################################
+# SETUP - Check environment
+################################################################################
+
+gpuci_logger "Check environment variables"
+env
+
+gpuci_logger "Activate conda env"
+. /opt/conda/etc/profile.d/conda.sh
+conda activate rapids
+
+# Remove rapidsai-nightly channel if we are building main branch
+if [ "$SOURCE_BRANCH" = "main" ]; then
+  conda config --system --remove channels rapidsai-nightly
+fi
+
+gpuci_logger "Check versions"
+python --version
+$CC --version
+$CXX --version
+
+gpuci_logger "Check conda environment"
+conda info
+conda config --show-sources
+conda list --show-channel-urls
+
+# FIX Added to deal with Anancoda SSL verification issues during conda builds
+conda config --set ssl_verify False
+
+# FIXME: for now, force the building of all packages so they are built on a
+# machine with a single CUDA version, then have the gpu/build.sh script simply
+# install. This should eliminate a mismatch between different CUDA versions on
+# cpu vs. gpu builds that is problematic with CUDA 11.5 Enhanced Compat.
+if [ "$BUILD_LIBRAFT" == '1' ]; then
+  BUILD_PYRAFT=1
+  # If we are doing CUDA + Python builds, libraft package is located at ${CONDA_BLD_DIR}
+  CONDA_LOCAL_CHANNEL="${CONDA_BLD_DIR}"
+else
+  # If we are doing Python builds only, libraft package is placed here by Project Flash
+  CONDA_LOCAL_CHANNEL="ci/artifacts/raft/cpu/.conda-bld/"
+fi
+
+
+###############################################################################
+# BUILD - Conda package builds
+###############################################################################
+
+if [ "$BUILD_LIBRAFT" == '1' ]; then
+  gpuci_logger "Building conda packages for libraft-nn, libraft-distance, and libraft-headers"
+  if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft-nn
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft-distance
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft-headers
+  else
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft-nn
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft-distance
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft-headers
+    mkdir -p ${CONDA_BLD_DIR}/libraft
+    mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft/work
+  fi
+else
+  gpuci_logger "SKIPPING build of conda packages for libraft-nn, libraft-distance and libraft-headers"
+fi
+
+if [ "$BUILD_raft" == "1" ]; then
+  gpuci_logger "Building conda packages for pyraft"
+  if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/pyraft --python=$PYTHON
+  else
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/pyraft -c ${CONDA_LOCAL_CHANNEL} --dirty --no-remove-work-dir --python=$PYTHON
+    mkdir -p ${CONDA_BLD_DIR}/pyraft
+    mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/pyraft/work
+  fi
+else
+  gpuci_logger "SKIPPING build of conda packages for pyraft"
+fi
+
+################################################################################
+# UPLOAD - Conda packages
+################################################################################
+
+gpuci_logger "Upload conda packages"
+source ci/cpu/upload.sh
diff --git a/ci/cpu/prebuild.sh b/ci/cpu/prebuild.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Copyright (c) 2022, NVIDIA CORPORATION.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
+    #If project flash is not activate, always build both
+    export BUILD_PYRAFT=1
+    export BUILD_LIBRAFT=1
+fi
+
+export UPLOAD_LIBRAFT=1
diff --git a/ci/cpu/upload.sh b/ci/cpu/upload.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+# Copyright (c) 2022, NVIDIA CORPORATION.
+#
+# Adopted from https://github.com/tmcdonell/travis-scripts/blob/dfaac280ac2082cd6bcaba3217428347899f2975/update-accelerate-buildbot.sh
+
+set -e
+
+# Setup 'gpuci_retry' for upload retries (results in 4 total attempts)
+export GPUCI_RETRY_MAX=3
+export GPUCI_RETRY_SLEEP=30
+
+# Set default label options if they are not defined elsewhere
+export LABEL_OPTION=${LABEL_OPTION:-"--label main"}
+
+# Skip uploads unless BUILD_MODE == "branch"
+if [ ${BUILD_MODE} != "branch" ]; then
+  echo "Skipping upload"
+  return 0
+fi
+
+# Skip uploads if there is no upload key
+if [ -z "$MY_UPLOAD_KEY" ]; then
+  echo "No upload key"
+  return 0
+fi
+
+################################################################################
+# SETUP - Get conda file output locations
+################################################################################
+
+gpuci_logger "Get conda file output locations"
+
+export LIBRAFT_NN_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_nn --output`
+export LIBRAFT_DISTANCE_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_distance --output`
+export LIBRAFT_HEADERS_FILE=`conda build --croot ${CONDA_BLD_DIR} conda/recipes/libraft_headers --output`
+export PYRAFT_FILE=`conda build --croot ${CONDA_BLD_DIR} conda/recipes/pyraft --python=$PYTHON --output`
+
+################################################################################
+# UPLOAD - Conda packages
+################################################################################
+
+gpuci_logger "Starting conda uploads"
+
+if [[ "$BUILD_LIBRAFT" == "1" && "$UPLOAD_LIBRAFT" == "1" ]]; then
+  # libraft-nn
+  test -e ${LIBRAFT_NN_FILE}
+  echo "Upload libraft-nn"
+  echo ${LIBRAFT_NN_FILE}
+  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_NN_FILE} --no-progress
+
+  # libraft-distance
+  test -e ${LIBRAFT_DISTANCE_FILE}
+  echo "Upload libraft-distance"
+  echo ${LIBRAFT_DISTANCE_FILE}
+  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_DISTANCE_FILE} --no-progress
+
+  # libraft-headers
+  test -e ${LIBRAFT_HEADERS_FILE}
+  echo "Upload libraft-nn"
+  echo ${LIBRAFT_HEADERS_FILE}
+  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_HEADERS_FILE} --no-progress
+fi
+
+if [[ "$BUILD_PYRAFT" == "1" ]]; then
+  test -e ${PYRAFT_FILE}
+  echo "Upload pyraft"
+  echo ${PYRAFT_FILE}
+  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${PYRAFT_FILE} --no-progress
+fi