Skip to content

Commit

Permalink
feat(actions): updates to gha workflows (#11150)
Browse files Browse the repository at this point in the history
  • Loading branch information
@david-leifker
david-leifker authored Aug 14, 2024
1 parent 897173f commit 18ce104
Show file tree
Hide file tree
Showing 12 changed files with 93 additions and 45 deletions.
7 changes: 4 additions & 3 deletions .github/actions/ci-optimization/action.yml
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
name: 'Identify CI Optimizations'
description: 'Determine if code changes are specific to certain modules.'
name: "Identify CI Optimizations"
description: "Determine if code changes are specific to certain modules."

outputs:
frontend-only:
Expand Down Expand Up @@ -44,9 +44,10 @@ outputs:
runs:
using: "composite"
steps:
- uses: dorny/paths-filter@v2
- uses: dorny/paths-filter@v3
id: filter
with:
token: "" # Empty token forces it to use raw git commands.
filters: |
frontend:
- "datahub-frontend/**"
Expand Down
31 changes: 20 additions & 11 deletions .github/actions/docker-custom-build-and-push/action.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,13 @@ inputs:
build-args:
description: "List of build-time variables. Same as docker/build-push-action"
required: false
tags:
# e.g. latest,head,sha12345
description: "List of tags to use for the Docker image"
image_tag:
# e.g. pr12345 OR head OR v0.1.2.3
description: "Main tag to use for the Docker image"
required: true
flavor:
description: 'Image flavor (e.g., slim, full)'
required: false
target:
description: "Sets the target stage to build"
required: false
Expand All @@ -45,13 +48,17 @@ runs:
steps:
- name: Docker meta
id: docker_meta
uses: crazy-max/ghaction-docker-meta@v1
uses: docker/metadata-action@v5
with:
# list of Docker images to use as base name for tags
images: ${{ inputs.images }}
# add git short SHA as Docker tag
tag-custom: ${{ inputs.tags }}
tag-custom-only: true
flavor: |
latest=false
suffix=${{ inputs.flavor && format('-{0}', inputs.flavor) || '' }}
tags: |
type=raw,value=${{ inputs.image_tag }}
type=raw,value=head,enable=${{ github.ref == format('refs/heads/{0}', 'acryl-main') }}
type=ref,event=pr,prefix=pr
type=sha,prefix=,format=short
# Code for testing the build when not pushing to Docker Hub.
- name: Build and Load image for testing (if not publishing)
Expand All @@ -74,11 +81,13 @@ runs:
if: ${{ inputs.publish != 'true' }}
shell: bash
run: |
IMAGES="""
${{ inputs.images }}
"""
TAGS="""
${{ steps.docker_meta.outputs.tags }}
${{ inputs.image_tag }}
"""
echo "SINGLE_TAG=$(echo $TAGS | tr '\n' ' ' | awk -F' ' '{ print $1 }')" >> $GITHUB_OUTPUT
id: single_tag
echo "SINGLE_TAG=$(echo $IMAGES | tr '\n' ' ' | awk -F' ' '{ print $1 }'):$(echo $TAGS | tr '\n' ' ' | awk -F' ' '{ print $1 }')" >> $GITHUB_OUTPUT
- name: Upload image locally for testing (if not publishing)
uses: ishworkh/docker-image-artifact-upload@v1
if: ${{ inputs.publish != 'true' }}
Expand Down
12 changes: 10 additions & 2 deletions .github/scripts/docker_helpers.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,14 @@ export MAIN_BRANCH="master"
export MAIN_BRANCH_TAG="head"

function get_short_sha {
echo $(git rev-parse --short "$GITHUB_SHA")
echo $(git rev-parse --short "$GITHUB_SHA"|head -c7)
}

export SHORT_SHA=$(get_short_sha)
echo "SHORT_SHA: $SHORT_SHA"

function get_tag {
echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g'),${SHORT_SHA}
echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
}

function get_tag_slim {
Expand All @@ -38,3 +38,11 @@ function get_unique_tag_slim {
function get_unique_tag_full {
echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-full,g" -e 's,refs/tags/\(.*\),\1-full,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g')
}

function get_platforms_based_on_branch {
if [ "${{ github.event_name }}" == 'push' && "${{ github.ref }}" == "refs/heads/${MAIN_BRANCH}" ]; then
echo "linux/amd64,linux/arm64"
else
echo "linux/amd64"
fi
}
5 changes: 5 additions & 0 deletions .github/workflows/build-and-test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,11 @@ jobs:
timeout-minutes: 60
needs: setup
steps:
- name: Free up disk space
run: |
sudo apt-get remove 'dotnet-*' azure-cli || true
sudo rm -rf /usr/local/lib/android/ || true
sudo docker image prune -a -f || true
- uses: szenius/[email protected]
with:
timezoneLinux: ${{ matrix.timezone }}
Expand Down
51 changes: 25 additions & 26 deletions .github/workflows/docker-unified.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@ jobs:
publish: ${{ steps.publish.outputs.publish }}
pr-publish: ${{ steps.pr-publish.outputs.publish }}
python_release_version: ${{ steps.tag.outputs.python_release_version }}
short_sha: ${{ steps.tag.outputs.short_sha }}
branch_name: ${{ steps.tag.outputs.branch_name }}
repository_name: ${{ steps.tag.outputs.repository_name }}
frontend_change: ${{ steps.ci-optimize.outputs.frontend-change == 'true' }}
Expand Down Expand Up @@ -157,7 +156,7 @@ jobs:
with:
images: |
${{ env.DATAHUB_GMS_IMAGE }}
tags: ${{ needs.setup.outputs.tag }}
image_tag: ${{ needs.setup.outputs.tag }}
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
Expand Down Expand Up @@ -221,7 +220,7 @@ jobs:
with:
images: |
${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}
tags: ${{ needs.setup.outputs.tag }}
image_tag: ${{ needs.setup.outputs.tag }}
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
Expand Down Expand Up @@ -285,7 +284,7 @@ jobs:
with:
images: |
${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}
tags: ${{ needs.setup.outputs.tag }}
image_tag: ${{ needs.setup.outputs.tag }}
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
Expand Down Expand Up @@ -349,7 +348,7 @@ jobs:
with:
images: |
${{ env.DATAHUB_UPGRADE_IMAGE }}
tags: ${{ needs.setup.outputs.tag }}
image_tag: ${{ needs.setup.outputs.tag }}
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
Expand Down Expand Up @@ -394,7 +393,7 @@ jobs:
name: Build and Push DataHub Frontend Docker Image
runs-on: ubuntu-latest
needs: setup
if: ${{ needs.setup.outputs.frontend_change == 'true' || needs.setup.outputs.publish == 'true' }}
if: ${{ needs.setup.outputs.frontend_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true'}}
steps:
- name: Set up JDK 17
uses: actions/setup-java@v3
Expand All @@ -415,7 +414,7 @@ jobs:
with:
images: |
${{ env.DATAHUB_FRONTEND_IMAGE }}
tags: ${{ needs.setup.outputs.tag }}
image_tag: ${{ needs.setup.outputs.tag }}
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
Expand Down Expand Up @@ -469,7 +468,7 @@ jobs:
with:
images: |
${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}
tags: ${{ needs.setup.outputs.tag }}
image_tag: ${{ needs.setup.outputs.tag }}
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
Expand All @@ -490,7 +489,7 @@ jobs:
with:
images: |
${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}
tags: ${{ needs.setup.outputs.tag }}
image_tag: ${{ needs.setup.outputs.tag }}
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
Expand All @@ -511,7 +510,7 @@ jobs:
with:
images: |
${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}
tags: ${{ needs.setup.outputs.tag }}
image_tag: ${{ needs.setup.outputs.tag }}
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
Expand All @@ -525,7 +524,7 @@ jobs:
outputs:
tag: ${{ steps.tag.outputs.tag }}
needs: setup
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
steps:
- name: Check out the repo
uses: acryldata/sane-checkout-action@v3
Expand All @@ -536,7 +535,7 @@ jobs:
target: base
images: |
${{ env.DATAHUB_INGESTION_BASE_IMAGE }}
tags: ${{ needs.setup.outputs.tag }}
image_tag: ${{ needs.setup.outputs.tag }}
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
Expand All @@ -552,7 +551,7 @@ jobs:
outputs:
tag: ${{ steps.tag.outputs.tag }}
needs: [setup, datahub_ingestion_base_build]
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
steps:
- name: Check out the repo
uses: acryldata/sane-checkout-action@v3
Expand All @@ -574,7 +573,7 @@ jobs:
target: slim-install
images: |
${{ env.DATAHUB_INGESTION_BASE_IMAGE }}
tags: ${{ needs.setup.outputs.slim_tag }}
image_tag: ${{ needs.setup.outputs.slim_tag }}
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
build-args: |
Expand All @@ -593,7 +592,7 @@ jobs:
outputs:
tag: ${{ steps.tag.outputs.tag }}
needs: [setup, datahub_ingestion_base_build]
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
steps:
- name: Check out the repo
uses: acryldata/sane-checkout-action@v3
Expand Down Expand Up @@ -636,7 +635,7 @@ jobs:
tag: ${{ steps.tag.outputs.tag }}
needs_artifact_download: ${{ needs.setup.outputs.ingestion_change == 'true' && ( needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true') }}
needs: [setup, datahub_ingestion_base_slim_build]
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
steps:
- name: Set up JDK 17
uses: actions/setup-java@v3
Expand All @@ -647,7 +646,7 @@ jobs:
- name: Check out the repo
uses: acryldata/sane-checkout-action@v3
- name: Build codegen
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish =='true' }}
run: ./gradlew :metadata-ingestion:codegen
- name: Download Base Image
uses: ishworkh/docker-image-artifact-download@v1
Expand All @@ -661,7 +660,7 @@ jobs:
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
- name: Build and push Slim Image
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
uses: ./.github/actions/docker-custom-build-and-push
with:
target: final
Expand All @@ -672,7 +671,7 @@ jobs:
DOCKER_VERSION=${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}
RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }}
APP_ENV=slim
tags: ${{ needs.setup.outputs.slim_tag }}
image_tag: ${{ needs.setup.outputs.slim_tag }}
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
Expand Down Expand Up @@ -723,7 +722,7 @@ jobs:
tag: ${{ steps.tag.outputs.tag }}
needs_artifact_download: ${{ needs.setup.outputs.ingestion_change == 'true' && ( needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' ) }}
needs: [setup, datahub_ingestion_base_full_build]
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
steps:
- name: Set up JDK 17
uses: actions/setup-java@v3
Expand All @@ -734,7 +733,7 @@ jobs:
- name: Check out the repo
uses: acryldata/sane-checkout-action@v3
- name: Build codegen
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
run: ./gradlew :metadata-ingestion:codegen
- name: Download Base Image
uses: ishworkh/docker-image-artifact-download@v1
Expand All @@ -748,7 +747,7 @@ jobs:
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
- name: Build and push Full Image
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
uses: ./.github/actions/docker-custom-build-and-push
with:
target: final
Expand All @@ -758,7 +757,7 @@ jobs:
BASE_IMAGE=${{ env.DATAHUB_INGESTION_BASE_IMAGE }}
DOCKER_VERSION=${{ needs.setup.outputs.ingestion_base_change == 'true' && needs.setup.outputs.unique_tag || 'head' }}
RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }}
tags: ${{ needs.setup.outputs.tag }}
image_tag: ${{ needs.setup.outputs.tag }}
username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
publish: ${{ needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
Expand All @@ -776,7 +775,7 @@ jobs:
name: "[Monitoring] Scan Datahub Ingestion images for vulnerabilities"
runs-on: ubuntu-latest
needs: [setup, datahub_ingestion_full_build]
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }}
if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true' }}
steps:
- name: Checkout # adding checkout step just to make trivy upload happy
uses: acryldata/sane-checkout-action@v3
Expand Down Expand Up @@ -965,7 +964,7 @@ jobs:
echo 'datahub-ingestion head-slim images'
docker pull '${{ env.DATAHUB_INGESTION_IMAGE }}:head-slim'
if [ '${{ needs.datahub_ingestion_slim_build.outputs.tag || 'head-slim' }}' != 'head-slim' ]; then
docker tag '${{ env.DATAHUB_INGESTION_IMAGE }}:head-slim' '${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }}'
docker tag '${{ env.DATAHUB_INGESTION_IMAGE }}:head-slim' '${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.setup.outputs.unique_tag }}'
fi
fi
- name: Disk Check
Expand Down Expand Up @@ -1049,7 +1048,7 @@ jobs:
runs-on: ubuntu-latest
needs: [setup, smoke_test]
steps:
- uses: aws-actions/configure-aws-credentials@v1
- uses: aws-actions/configure-aws-credentials@v4
if: ${{ needs.setup.outputs.publish != 'false' && github.repository_owner == 'datahub-project' && needs.setup.outputs.repository_name == 'datahub' }}
with:
aws-access-key-id: ${{ secrets.AWS_SQS_ACCESS_KEY_ID }}
Expand Down
5 changes: 5 additions & 0 deletions .github/workflows/lint-actions.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,3 +14,8 @@ jobs:
- uses: reviewdog/action-actionlint@v1
with:
reporter: github-pr-review
permissions:
contents: read
checks: write
pull-requests: write
issues: write
5 changes: 5 additions & 0 deletions .github/workflows/metadata-ingestion.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,11 @@ jobs:
- python-version: "3.10"
fail-fast: false
steps:
- name: Free up disk space
run: |
sudo apt-get remove 'dotnet-*' azure-cli || true
sudo rm -rf /usr/local/lib/android/ || true
sudo docker image prune -a -f || true
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
Expand Down
5 changes: 5 additions & 0 deletions .github/workflows/metadata-io.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,11 @@ jobs:
timeout-minutes: 60
needs: setup
steps:
- name: Free up disk space
run: |
sudo apt-get remove 'dotnet-*' azure-cli || true
sudo rm -rf /usr/local/lib/android/ || true
sudo docker image prune -a -f || true
- uses: acryldata/sane-checkout-action@v3
- name: Set up JDK 17
uses: actions/setup-java@v3
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/metadata-model.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ jobs:
run: ./gradlew :metadata-ingestion:modelDocGen
- name: Configure AWS Credentials
if: ${{ needs.setup.outputs.publish == 'true' }}
uses: aws-actions/configure-aws-credentials@v3
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.ACRYL_CI_ARTIFACTS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.ACRYL_CI_ARTIFACTS_ACCESS_KEY }}
Expand Down
Loading

0 comments on commit 18ce104

Please sign in to comment.