Skip to content

Commit ca47dd0

Browse files
eranhderanhd
committed
MOD-8197 Unified CI workflows (#1304)
- Updated and added new GitHub Actions workflows for build, test, and deployment - Introduced event-based workflows for specific triggers: - `event-ci.yml`: Runs CI checks on pull requests and pushes to specific branches - `event-nightly.yml`: Executes nightly builds and tests - `event-tag.yml`: Triggers workflows on new tags for release processes - Differentiated flow-based workflows for various environments: - `flow-alpine.yml`: Handles builds and tests for Alpine Linux - `flow-linux-x86.yml`: Manages workflows for x86 Linux distributions - `flow-macos.yml`: Manages workflows for macOS environments - `flow-ubuntu-arm.yml`: Handles builds and tests for Ubuntu on ARM architecture - Removed obsolete workflows - Added new installation scripts for various Linux distributions and macOS - Modified existing scripts and configurations for better compatibility and performance (cherry picked from commit fd0882b)
1 parent 43b950b commit ca47dd0

31 files changed

Lines changed: 1000 additions & 1246 deletions

File tree

‎.circleci/config.yml‎

Lines changed: 0 additions & 553 deletions
This file was deleted.

‎.github/actions/build-json-module-and-redis-with-cargo/action.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ runs:
1919
- name: Build module
2020
shell: bash
2121
run: |
22-
. "$HOME/.cargo/env"
22+
. $HOME/.cargo/env
23+
echo "source $HOME/.cargo/env" >> $HOME/.bash_profile
2324
cargo build --release
2425
cp $(realpath ./target/release)/librejson.so $(realpath ./target/release)/rejson.so

‎.github/actions/build-json-module-and-redis/action.yml‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,8 @@ description: |
33
Build JSON module and Redis Server
44
55
inputs:
6-
redis-ref:
7-
description: 'Redis version to build'
8-
required: true
9-
default: '7.2.1'
6+
sanitizer:
7+
type: string
108

119
runs:
1210
using: composite
@@ -19,4 +17,4 @@ runs:
1917
- name: Build module
2018
shell: bash
2119
run: |
22-
make build -j `nproc`
20+
make build -j `nproc`

‎.github/actions/make-pack/action.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
name: Run make pack module script
2+
3+
runs:
4+
using: composite
5+
steps:
6+
- name: Pack module
7+
shell: bash
8+
run: |
9+
make pack BRANCH=$TAG_OR_BRANCH

‎.github/actions/pack-module/action.yml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
name: Run pack module script
2+
3+
runs:
4+
using: composite
5+
steps:
6+
- name: Pack module
7+
shell: bash
8+
run: |
9+
if command -v scl_source &> /dev/null
10+
then
11+
. scl_source enable devtoolset-11 || true
12+
fi
13+
. venv/bin/activate
14+
git config --global --add safe.directory $GITHUB_WORKSPACE
15+
export PATH="$GITHUB_WORKSPACE/redis/src:$PATH"
16+
MODULE=$(realpath ./target/release/rejson.so) BRANCH=$TAG_OR_BRANCH \
17+
SHOW=1 OSNICK=${{ matrix.docker.nick }} ./sbin/pack.sh

‎.github/actions/san-run-tests/action.yml‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
name: Common Flow for Tests
2+
3+
# Documentation: https://redislabs.atlassian.net/wiki/spaces/DX/pages/3967844669/RediSearch+CI+refactor
4+
5+
inputs:
6+
env:
7+
default: "ubuntu-22.04"
8+
type: string
9+
container:
10+
type: string
11+
sanitizer:
12+
type: string
13+
test-config:
14+
description: 'Test configuration environment variable (e.g. "CONFIG=tls" or "QUICK=1")'
15+
required: true
16+
type: string
17+
redis-ref:
18+
description: 'Redis ref to checkout'
19+
type: string
20+
required: true
21+
22+
runs:
23+
using: composite
24+
steps:
25+
- name: Get Installation Mode
26+
shell: bash
27+
id: mode
28+
run: |
29+
[[ -z "${{ inputs.container }}" ]] && echo "mode=sudo" >> $GITHUB_OUTPUT || echo "mode=" >> $GITHUB_OUTPUT
30+
- name: Check if node20 is Supported
31+
id: node20
32+
uses: ./.github/actions/node20-supported
33+
with:
34+
container: ${{ inputs.container }}
35+
- name: Install git
36+
shell: bash
37+
run: |
38+
# TODO: must be changed to run a script based on the input env
39+
echo ::group::Install git
40+
${{ steps.mode.outputs.mode }} apt-get update && apt-get install -y git
41+
echo ::endgroup::
42+
- name: Setup specific
43+
shell: bash
44+
working-directory: .install
45+
run: |
46+
echo ::group::OS-Specific Setup
47+
./install_script.sh ${{ steps.mode.outputs.mode }}
48+
echo ::endgroup::
49+
echo ::group::Get Rust
50+
./getrust.sh ${{ steps.mode.outputs.mode }}
51+
echo ::endgroup::
52+
53+
- name: Full checkout (node20 supported)
54+
if: steps.node20.outputs.supported == 'true'
55+
uses: actions/checkout@v4
56+
with:
57+
submodules: recursive
58+
- name: Full checkout (node20 unsupported)
59+
if: steps.node20.outputs.supported == 'false'
60+
uses: actions/checkout@v4
61+
with:
62+
submodules: recursive
63+
64+
- name: Get Redis
65+
uses: actions/checkout@v4
66+
with:
67+
repository: redis/redis
68+
ref: ${{ inputs.redis-ref }}
69+
path: redis
70+
submodules: 'recursive'
71+
- name: Build
72+
uses: ./.github/actions/build-json-module-and-redis
73+
with:
74+
sanitizer: ${{ inputs.san }}
75+
76+
- name: Set Artifact Names
77+
shell: bash
78+
# Artifact names have to be unique, so we base them on the environment.
79+
# We also remove invalid characters from the name.
80+
id: artifact-names
81+
run: |
82+
# Invalid characters include: Double quote ", Colon :, Less than <, Greater than >, Vertical bar |, Asterisk *, Question mark ?
83+
echo "name=$(echo "${{ inputs.container || inputs.env }} ${{ runner.arch }}, Redis ${{ inputs.redis-ref }}" | \
84+
sed -e 's/[":\/\\<>\|*?]/_/g' -e 's/__*/_/g' -e 's/^_//' -e 's/_$//')" >> $GITHUB_OUTPUT
85+
86+
- name: Run tests
87+
shell: bash
88+
id: test
89+
run: |
90+
echo ::group::Activate virtual environment
91+
python3 -m venv venv
92+
echo "source $PWD/venv/bin/activate" >> ~/.bash_profile
93+
source venv/bin/activate
94+
echo ::endgroup::
95+
echo ::group::Install python dependencies
96+
./.install/common_installations.sh
97+
echo ::endgroup::
98+
echo ::group::Unit tests
99+
. "$HOME/.cargo/env"
100+
make cargo_test LOG=1 CLEAR_LOGS=0 SAN=${{ inputs.san }}
101+
echo ::endgroup::
102+
echo ::group::Flow tests
103+
make pytest LOG=1 CLEAR_LOGS=0 SAN=${{ inputs.san }} ${{ inputs.test-config }}
104+
echo ::endgroup::
105+
env:
106+
PIP_BREAK_SYSTEM_PACKAGES: 1
107+
108+
- name: Upload test artifacts (node20 supported)
109+
if: steps.node20.outputs.supported == 'true' && steps.test.outcome == 'failure'
110+
uses: actions/upload-artifact@v4
111+
with:
112+
name: Test logs ${{ steps.artifact-names.outputs.name }}
113+
path: tests/**/logs/*.log*
114+
if-no-files-found: ignore
115+
- name: Upload test artifacts (node20 unsupported)
116+
if: steps.node20.outputs.supported == 'false' && steps.test.outcome == 'failure'
117+
uses: actions/upload-artifact@v4
118+
with:
119+
name: Test logs ${{ steps.artifact-names.outputs.name }}
120+
path: tests/**/logs/*.log*
121+
if-no-files-found: ignore
122+
123+
- name: Fail flow if tests failed
124+
shell: bash
125+
if: steps.test.outcome == 'failure'
126+
run: exit 1

‎.github/actions/setup-env/action.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,4 +65,4 @@ runs:
6565
id: set-redis-ref
6666
run: |
6767
export REDIS_REF="${{ inputs.redis-ref || '8.0-m01'}}"
68-
echo "REDIS_REF=${REDIS_REF}" >> $GITHUB_OUTPUT
68+
echo "REDIS_REF=${REDIS_REF}" >> $GITHUB_OUTPUT

‎.github/actions/upload-artifacts-to-s3-without-make/action.yml‎

Lines changed: 32 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,6 @@ inputs:
99
aws-secret-access-key:
1010
description: 'AWS Secret Access Key'
1111
required: true
12-
github-ref:
13-
description: 'GitHub ref'
14-
required: true
1512
osnick:
1613
description: 'OS Nickname'
1714
required: false
@@ -20,29 +17,40 @@ inputs:
2017
runs:
2118
using: composite
2219
steps:
23-
- name: Configure AWS credentials
24-
uses: aws-actions/configure-aws-credentials@v3
25-
with: # todo: use role instead of access key
26-
aws-access-key-id: ${{ inputs.aws-access-key-id }}
27-
aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
28-
aws-region: "us-east-1"
29-
- name: Upload artifacts to S3 - staging
20+
- name: Configure AWS credentials and upload artifcats # todo - use aws role instead
3021
shell: bash
3122
run: |
32-
echo ::group::install aws cli
23+
echo ::group::install aws cli
3324
python3 -m venv .aws-cli-venv && source .aws-cli-venv/bin/activate &&
3425
pip3 install --upgrade pip && pip3 install --no-cache-dir awscli && rm -rf /var/cache/apk/*
35-
echo ::endgroup::
36-
echo ::group::upload artifacts
37-
SNAPSHOT=1 SHOW=1 VERBOSE=1 ./sbin/upload-artifacts
38-
echo ::endgroup::
39-
echo ::group::upload staging release
40-
RELEASE=1 SHOW=1 STAGING=1 VERBOSE=1 ./sbin/upload-artifacts
41-
echo ::endgroup::
42-
43-
echo ::group::upload production release
44-
# todo: trigger this manually instead
45-
if [[ "${{ inputs.github-ref}}" != 'refs/heads/master' ]]; then
46-
RELEASE=1 SHOW=1 VERBOSE=1 ./sbin/upload-artifacts
26+
echo ::endgroup::
27+
28+
# Variables from the workflow
29+
export AWS_ACCESS_KEY_ID="${{ inputs.aws-access-key-id }}"
30+
export AWS_SECRET_ACCESS_KEY="${{ inputs.aws-secret-access-key }}"
31+
export AWS_REGION="us-east-1"
32+
# Check if the required environment variables are set
33+
if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ] || [ -z "$AWS_REGION" ]; then
34+
echo "Missing AWS credentials or region configuration."
35+
exit 1
4736
fi
48-
echo ::endgroup::
37+
# Configure AWS CLI with provided credentials and region
38+
echo "Configuring AWS CLI with access keys..."
39+
aws configure set aws_access_key_id "$AWS_ACCESS_KEY_ID"
40+
aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY"
41+
aws configure set region "$AWS_REGION"
42+
43+
echo ::group::upload artifacts
44+
SNAPSHOT=1 SHOW=1 VERBOSE=1 ./sbin/upload-artifacts
45+
echo ::endgroup::
46+
echo ::group::upload staging release
47+
RELEASE=1 SHOW=1 STAGING=1 VERBOSE=1 ./sbin/upload-artifacts
48+
echo ::endgroup::
49+
50+
echo ::group::upload production release
51+
REF="${{ inputs.github-ref }}"
52+
PATTERN="refs/tags/v[0-9]+.*"
53+
if [[ $REF =~ $PATTERN ]]; then
54+
RELEASE=1 SHOW=1 VERBOSE=1 ./sbin/upload-artifacts
55+
fi
56+
echo ::endgroup::

‎.github/actions/upload-artifacts-to-s3/action.yml‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,19 +20,29 @@ inputs:
2020
runs:
2121
using: composite
2222
steps:
23-
- name: Configure AWS credentials
24-
uses: aws-actions/configure-aws-credentials@v3
25-
with: # todo: use role instead of access key
26-
aws-access-key-id: ${{ inputs.aws-access-key-id }}
27-
aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
28-
aws-region: "us-east-1"
2923
- name: Upload artifacts to S3 - staging
3024
shell: bash
3125
run: |
3226
echo ::group::install aws cli
3327
python3 -m venv .aws-cli-venv && source .aws-cli-venv/bin/activate &&
3428
pip3 install --upgrade pip && pip3 install --no-cache-dir awscli && rm -rf /var/cache/apk/*
3529
echo ::endgroup::
30+
31+
# Variables from the workflow
32+
export AWS_ACCESS_KEY_ID="${{ inputs.aws-access-key-id }}"
33+
export AWS_SECRET_ACCESS_KEY="${{ inputs.aws-secret-access-key }}"
34+
export AWS_REGION="us-east-1"
35+
# Check if the required environment variables are set
36+
if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ] || [ -z "$AWS_REGION" ]; then
37+
echo "Missing AWS credentials or region configuration."
38+
exit 1
39+
fi
40+
# Configure AWS CLI with provided credentials and region
41+
echo "Configuring AWS CLI with access keys..."
42+
aws configure set aws_access_key_id "$AWS_ACCESS_KEY_ID"
43+
aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY"
44+
aws configure set region "$AWS_REGION"
45+
3646
echo ::group::upload artifacts
3747
make upload-artifacts SHOW=1 VERBOSE=1
3848
echo ::endgroup::
@@ -45,4 +55,4 @@ runs:
4555
if [[ "${{ inputs.github-ref}}" != 'refs/heads/master' ]]; then
4656
make upload-release SHOW=1 VERBOSE=1
4757
fi
48-
echo ::endgroup::
58+
echo ::endgroup::

‎.github/actions/upload-artifacts/action.yml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
name: Pack the module and upload it to S3
2+
description: Pack the module and upload it to S3
3+
4+
inputs:
5+
image: # The Docker image to use for the build
6+
description: 'The Docker image to use for the build'
7+
required: true
8+
9+
runs:
10+
using: composite
11+
steps:
12+
- name: Set Artifact Names
13+
# Artifact names have to be unique, so we base them on the environment.
14+
# We also remove invalid characters from the name.
15+
id: artifact-names
16+
shell: bash
17+
run: | # Invalid characters include: Double quote ", Colon :, Less than <, Greater than >, Vertical bar |, Asterisk *, Question mark ?
18+
echo "name=$(echo "${{ inputs.image }} x86-64, Redis unstable" | \
19+
sed -e 's/[":\/\\<>\|*?]/_/g' -e 's/__*/_/g' -e 's/^_//' -e 's/_$//')" >> $GITHUB_OUTPUT
20+
- name: Upload test artifacts
21+
if: inputs.image != 'amazonlinux:2' && inputs.image != 'ubuntu:bionic'
22+
uses: actions/upload-artifact@v4
23+
with:
24+
name: Test logs ${{ steps.artifact-names.outputs.name }}
25+
path: tests/**/logs/*.log*
26+
if-no-files-found: ignore

0 commit comments

Comments
 (0)