Merge branch 'master' into patch-7

datahub-project · Aug 8, 2022 · 4b5a095 · 4b5a095
2 parents 50f8275 + 33339e2
commit 4b5a095
Show file tree

Hide file tree

Showing 796 changed files with 94,661 additions and 61,728 deletions.
diff --git a/.github/actions/docker-custom-build-and-push/action.yml b/.github/actions/docker-custom-build-and-push/action.yml
@@ -0,0 +1,92 @@
+name: Custom Docker build and push
+description: "Build and push a Docker image to Docker Hub"
+
+inputs:
+  username:
+    description: "Docker Hub username"
+  password:
+    description: "Docker Hub password"
+  publish:
+    description: "Set to true to actually publish the image to Docker Hub"
+
+  context:
+    description: "Same as docker/build-push-action"
+    required: false
+  file:
+    description: "Same as docker/build-push-action"
+    required: false
+  platforms:
+    description: "Same as docker/build-push-action"
+    required: false
+
+  images:
+    # e.g. linkedin/datahub-gms
+    description: "List of Docker images to use as base name for tags"
+    required: true
+  tags:
+    # e.g. latest,head,sha12345
+    description: "List of tags to use for the Docker image"
+    required: true
+outputs:
+  image_tag:
+    description: "Docker image tags"
+    value: ${{ steps.docker_meta.outputs.tags }}
+  # image_name: ${{ env.DATAHUB_GMS_IMAGE }}
+
+runs:
+  using: "composite"
+
+  steps:
+    - name: Docker meta
+      id: docker_meta
+      uses: crazy-max/ghaction-docker-meta@v1
+      with:
+        # list of Docker images to use as base name for tags
+        images: ${{ inputs.images }}
+        # add git short SHA as Docker tag
+        tag-custom: ${{ inputs.tags }}
+        tag-custom-only: true
+
+    # Code for testing the build when not pushing to Docker Hub.
+    - name: Build and Load image for testing (if not publishing)
+      uses: docker/build-push-action@v2
+      if: ${{ inputs.publish != 'true' }}
+      with:
+        context: ${{ inputs.context }}
+        file: ${{ inputs.file }}
+        # TODO this only does single-platform builds in testing?
+        # leaving it for now since it matches the previous behavior
+        platforms: linux/amd64
+        tags: ${{ steps.docker_meta.outputs.tags }}
+        load: true
+        push: false
+    - name: Upload image locally for testing (if not publishing)
+      uses: ishworkh/docker-image-artifact-upload@v1
+      if: ${{ inputs.publish != 'true' }}
+      with:
+        image: ${{ steps.docker_meta.outputs.tags }}
+
+    # Code for building multi-platform images and pushing to Docker Hub.
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v1
+      if: ${{ inputs.publish == 'true' }}
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+      if: ${{ inputs.publish == 'true' }}
+    - name: Login to DockerHub
+      uses: docker/login-action@v1
+      if: ${{ inputs.publish == 'true' }}
+      with:
+        username: ${{ inputs.username }}
+        password: ${{ inputs.password }}
+    - name: Build and Push Multi-Platform image
+      uses: docker/build-push-action@v2
+      if: ${{ inputs.publish == 'true' }}
+      with:
+        context: ${{ inputs.context }}
+        file: ${{ inputs.file }}
+        platforms: ${{ inputs.platforms }}
+        tags: ${{ steps.docker_meta.outputs.tags }}
+        push: true
+
+    # TODO add code for vuln scanning?
diff --git a/.github/actions/ensure-codegen-updated/action.yml b/.github/actions/ensure-codegen-updated/action.yml
@@ -0,0 +1,16 @@
+name: 'Ensure codegen is updated'
+description: 'Will check the local filesystem against git, and abort if there are uncommitted changes.'
+
+runs:
+  using: "composite"
+  steps:
+    - shell: bash
+      run: |
+        if output=$(git status --porcelain) && [ ! -z "$output" ]; then
+          # See https://unix.stackexchange.com/a/155077/378179.
+          echo 'There are uncommitted changes:'
+          echo $output
+          exit 1
+        else
+          echo 'All good!'
+        fi
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -10,7 +10,6 @@ on:
     branches:
       - master
     paths-ignore:
-      - "docker/**"
       - "docs/**"
       - "**.md"
   release:
@@ -49,15 +48,7 @@ jobs:
             **/build/test-results/test/**
             **/junit.*.xml
       - name: Ensure codegen is updated
-        run: |
-          if output=$(git status --porcelain) && [ ! -z "$output" ]; then
-            # See https://unix.stackexchange.com/a/155077/378179.
-            echo 'There are uncommitted changes:'
-            echo $output
-            exit 1
-          else
-            echo 'All good!'
-          fi
+        uses: ./.github/actions/ensure-codegen-updated
       - name: Slack failure notification
         if: failure() && github.event_name == 'push'
         uses: kpritam/slack-job-status-action@v1

diff --git a/.github/workflows/check-datahub-jars.yml b/.github/workflows/check-datahub-jars.yml
@@ -5,6 +5,7 @@ on:
     branches:
       - master
     paths-ignore:
+      - "docker/**"
       - "docs/**"
       - "**.md"
   pull_request:

diff --git a/.github/workflows/check-quickstart.yml b/.github/workflows/check-quickstart.yml
@@ -0,0 +1,47 @@
+name: check quickstart
+on:
+  push:
+    branches:
+      - master
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+
+jobs:
+  test-quickstart:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-20.04]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/setup-python@v2
+      with:
+        python-version: "3.9.9"
+    - name: Install acryl-datahub
+      run: |
+        pip install --upgrade acryl-datahub
+        datahub version
+        python  -c "import platform; print(platform.platform())"
+    - name: Run quickstart
+      run: |
+        datahub docker quickstart
+    - name: Ingest sample data
+      run: |
+        datahub docker ingest-sample-data
+    - name: See status
+      run: |
+        docker ps -a && datahub docker check
+    - name: store logs
+      if: failure()
+      run: |
+        docker logs datahub-gms >& quickstart-gms.log
+    - name: Upload logs
+      uses: actions/upload-artifact@v2
+      if: failure()
+      with:
+        name: docker-quickstart-logs-${{ matrix.os }}
+        path: "*.log"
diff --git a/.github/workflows/close-stale-issues.yml b/.github/workflows/close-stale-issues.yml
@@ -0,0 +1,22 @@
+name: Close inactive issues
+on:
+  schedule:
+    - cron: "30 1 * * *"
+
+jobs:
+  close-issues:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v5
+        with:
+          days-before-issue-stale: 30
+          days-before-issue-close: 30
+          stale-issue-label: "stale"
+          stale-issue-message: "This issue is stale because it has been open for 30 days with no activity. If you believe this is still an issue on the latest DataHub release please leave a comment with the version that you tested it with. If this is a question/discussion please head to https://slack.datahubproject.io. For feature requests please use https://feature-requests.datahubproject.io"
+          close-issue-message: "This issue was closed because it has been inactive for 30 days since being marked as stale."
+          days-before-pr-stale: -1
+          days-before-pr-close: -1
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/docker-feast-source.yml b/.github/workflows/docker-feast-source.yml
@@ -3,21 +3,22 @@ on:
   push:
     branches:
       - master
-    paths-ignore:
-      - 'docs/**'
-      - '**.md'
+    paths:
+      - 'metadata-ingestion/src/datahub/ingestion/source/feast_image/**'
+      - '.github/workflows/docker-feast-source.yml'
   pull_request:
     branches:
       - master
     paths:
       - 'metadata-ingestion/src/datahub/ingestion/source/feast_image/**'
       - '.github/workflows/docker-feast-source.yml'
-    paths_ignore:
-      - '**.md'
-      - '**.env'
   release:
     types: [published, edited]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   setup:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/docker-ingestion-base.yml b/.github/workflows/docker-ingestion-base.yml
@@ -1,10 +1,13 @@
 name: ingestion base
 on:
+  release:
+    types: [published, edited]
   push:
     branches:
       - master
     paths:
       - "docker/datahub-ingestion/**"
+      - "gradle*"
   workflow_dispatch:
 
 concurrency:

diff --git a/.github/workflows/docker-ingestion-smoke.yml b/.github/workflows/docker-ingestion-smoke.yml
@@ -0,0 +1,42 @@
+name: ingestion smoke
+on:
+  release:
+    types: [published, edited]
+  push:
+    branches:
+      - master
+    paths:
+      - "docker/datahub-ingestion/**"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+
+  build-smoke:
+    name: Build and Push Docker Image to Docker Hub
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
+          password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
+      - name: Build and Push image
+        uses: docker/build-push-action@v2
+        with:
+          context: ./docker/datahub-ingestion
+          file: ./docker/datahub-ingestion/smoke.Dockerfile
+          platforms: linux/amd64,linux/arm64
+          tags: acryldata/datahub-ingestion-base:smoke
+          push: true
diff --git a/.github/workflows/docker-ingestion.yml b/.github/workflows/docker-ingestion.yml
@@ -10,14 +10,17 @@ on:
     branches:
       - master
     paths:
-      - "docker/**"
+      - "metadata-ingestion/**"
+      - "metadata-models/**"
+      - "docker/datahub-ingestion/**"
       - ".github/workflows/docker-ingestion.yml"
-    paths_ignore:
-      - "**.md"
-      - "**.env"
   release:
     types: [published, edited]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   setup:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/docker-postgres-setup.yml b/.github/workflows/docker-postgres-setup.yml
@@ -3,9 +3,9 @@ on:
   push:
     branches:
       - master
-    paths-ignore:
-      - 'docs/**'
-      - '**.md'
+    paths:
+      - './docker/postgres-setup/**'
+      - '.github/workflows/docker-postgres-setup.yml'
   pull_request:
     branches:
       - master
@@ -15,6 +15,10 @@ on:
   release:
     types: [published, edited]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   setup:
     runs-on: ubuntu-latest