Merge branch 'main' into spacy-3-dot-0-support

Dazu-io · Feb 26, 2021 · c54751a · c54751a
2 parents 32ba805 + b13e70b
commit c54751a
Show file tree

Hide file tree

Showing 129 changed files with 3,865 additions and 1,289 deletions.
diff --git a/.github/runner/github-runner-deployment.yaml.tmpl b/.github/runner/github-runner-deployment.yaml.tmpl
@@ -25,7 +25,7 @@ spec:
       terminationGracePeriodSeconds: 180
       containers:
       - name: github-runner
-        image: us.gcr.io/rasa-platform/github-runner:latest
+        image: {{getenv "GH_RUNNER_IMAGE"}}:latest
         imagePullPolicy: Always
         livenessProbe:
           initialDelaySeconds: 30
@@ -44,7 +44,7 @@ spec:
             nvidia.com/gpu: 1
           requests:
             nvidia.com/gpu: 1
-            memory: 10Gi
+            memory: 10G
         env:
         - name: POD_NAME
           valueFrom:

diff --git a/.github/workflows/ci-model-regression-on-schedule.yml b/.github/workflows/ci-model-regression-on-schedule.yml
@@ -7,7 +7,7 @@ on:
     - cron:  '1 23 * * */7'
 
 env:
-  GKE_ZONE: us-central1-a
+  GKE_ZONE: us-central1
   GCLOUD_VERSION: "318.0.0"
 
 jobs:
@@ -49,18 +49,19 @@ jobs:
 
       - name: Render deployment template
         run: |-
+          export GH_RUNNER_IMAGE=${{ secrets.GH_RUNNER_IMAGE }}
           ./gomplate -f .github/runner/github-runner-deployment.yaml.tmpl -o runner_deployment.yaml
 
       # Setup gcloud CLI
       - uses: GoogleCloudPlatform/github-actions@e23988b2af9696c66e87d1efbc688d3a80c3be14
         with:
           version: "${{ env.GCLOUD_VERSION }}"
-          service_account_key: ${{ secrets.GKE_SA_RASA_CI_GPU }}
-          service_account_email: ${{ secrets.GKE_RASA_CI_GPU_SA_NAME }}
+          service_account_key: ${{ secrets.GKE_SA_RASA_CI_CD_GPU_RASA_CI_CD }}
+          service_account_email: ${{ secrets.GKE_RASA_CI_GPU_SA_NAME_RASA_CI_CD }}
 
       # Get the GKE credentials so we can deploy to the cluster
       - run: |-
-          gcloud container clusters get-credentials "${{ secrets.GKE_GPU_CLUSTER }}" --zone "$GKE_ZONE" --project "${{ secrets.GKE_SA_RASA_CI_GPU_PROJECT }}"
+          gcloud container clusters get-credentials "${{ secrets.GKE_GPU_CLUSTER_RASA_CI_CD }}" --zone "$GKE_ZONE" --project "${{ secrets.GKE_SA_RASA_CI_GPU_PROJECT_RASA_CI_CD }}"
 
       - name: Deploy Github Runner
         run: |-
@@ -220,6 +221,42 @@ jobs:
           name: report.json
           path: ./report.json
 
+      - name: Notify Slack of Failure 😱
+        if: failure()
+        uses: 8398a7/action-slack@v3
+        with:
+          status: custom
+          fields: workflow,job,commit,repo,ref,author,took
+          custom_payload: |
+            {
+              text: "Model regression tests CI",
+              attachments: [{
+                fallback: 'fallback',
+                color: '${{ job.status }}' === 'success' ? 'good' : '${{ job.status }}' === 'failure' ? 'danger' : 'warning',
+                title: `${process.env.AS_WORKFLOW}`,
+                text: 'Scheduled model regression test failed',
+                fields: [{
+                  title: 'Configuration',
+                  value: '${{ matrix.config }}',
+                  short: false
+                },
+                {
+                  title: 'Dataset',
+                  value: '${{ matrix.dataset }}',
+                  short: false
+                },
+                {
+                  title: 'GitHub Action',
+                  value: `https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}`,
+                  short: true
+                }],
+                actions: [{
+                }]
+              }]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_CI_MODEL_REGRESSION_TEST }}
+
   remove_runner_gpu:
     name: Delete Github Runner - GPU
     if: always()
@@ -233,12 +270,12 @@ jobs:
       - uses: GoogleCloudPlatform/github-actions@e23988b2af9696c66e87d1efbc688d3a80c3be14
         with:
           version: "${{ env.GCLOUD_VERSION }}"
-          service_account_key: ${{ secrets.GKE_SA_RASA_CI_GPU }}
-          service_account_email: ${{ secrets.GKE_RASA_CI_GPU_SA_NAME }}
+          service_account_key: ${{ secrets.GKE_SA_RASA_CI_CD_GPU_RASA_CI_CD }}
+          service_account_email: ${{ secrets.GKE_RASA_CI_GPU_SA_NAME_RASA_CI_CD }}
 
       # Get the GKE credentials so we can deploy to the cluster
       - run: |-
-          gcloud container clusters get-credentials "${{ secrets.GKE_GPU_CLUSTER }}" --zone "$GKE_ZONE" --project "${{ secrets.GKE_SA_RASA_CI_GPU_PROJECT }}"
+          gcloud container clusters get-credentials "${{ secrets.GKE_GPU_CLUSTER_RASA_CI_CD }}" --zone "$GKE_ZONE" --project "${{ secrets.GKE_SA_RASA_CI_GPU_PROJECT_RASA_CI_CD }}"
 
       - name: Remove Github Runner
         run: kubectl -n github-runner delete deployments github-runner-${GITHUB_RUN_ID} --grace-period=30
diff --git a/.github/workflows/ci-model-regression.yml b/.github/workflows/ci-model-regression.yml
@@ -11,7 +11,7 @@ on:
     types: [opened, synchronize, labeled]
 
 env:
-  GKE_ZONE: us-central1-a
+  GKE_ZONE: us-central1
   GCLOUD_VERSION: "318.0.0"
 
 jobs:
@@ -151,18 +151,19 @@ jobs:
 
       - name: Render deployment template
         run: |-
+          export GH_RUNNER_IMAGE=${{ secrets.GH_RUNNER_IMAGE }}
           gomplate -f .github/runner/github-runner-deployment.yaml.tmpl -o runner_deployment.yaml
 
       # Setup gcloud CLI
       - uses: GoogleCloudPlatform/github-actions@e23988b2af9696c66e87d1efbc688d3a80c3be14
         with:
           version: "${{ env.GCLOUD_VERSION }}"
-          service_account_key: ${{ secrets.GKE_SA_RASA_CI_GPU }}
-          service_account_email: ${{ secrets.GKE_RASA_CI_GPU_SA_NAME}}
+          service_account_key: ${{ secrets.GKE_SA_RASA_CI_CD_GPU_RASA_CI_CD }}
+          service_account_email: ${{ secrets.GKE_RASA_CI_GPU_SA_NAME_RASA_CI_CD }}
 
       # Get the GKE credentials so we can deploy to the cluster
       - run: |-
-          gcloud container clusters get-credentials "${{ secrets.GKE_GPU_CLUSTER }}" --zone "$GKE_ZONE" --project "${{ secrets.GKE_SA_RASA_CI_GPU_PROJECT }}"
+          gcloud container clusters get-credentials "${{ secrets.GKE_GPU_CLUSTER_RASA_CI_CD }}" --zone "$GKE_ZONE" --project "${{ secrets.GKE_SA_RASA_CI_GPU_PROJECT_RASA_CI_CD }}"
 
       - name: Deploy Github Runner
         run: |-
@@ -621,12 +622,12 @@ jobs:
       - uses: GoogleCloudPlatform/github-actions@e23988b2af9696c66e87d1efbc688d3a80c3be14
         with:
           version: "${{ env.GCLOUD_VERSION }}"
-          service_account_key: ${{ secrets.GKE_SA_RASA_CI_GPU }}
-          service_account_email: ${{ secrets.GKE_RASA_CI_GPU_SA_NAME }}
+          service_account_key: ${{ secrets.GKE_SA_RASA_CI_CD_GPU_RASA_CI_CD }}
+          service_account_email: ${{ secrets.GKE_RASA_CI_GPU_SA_NAME_RASA_CI_CD }}
 
       # Get the GKE credentials so we can deploy to the cluster
       - run: |-
-          gcloud container clusters get-credentials "${{ secrets.GKE_GPU_CLUSTER }}" --zone "$GKE_ZONE" --project "${{ secrets.GKE_SA_RASA_CI_GPU_PROJECT }}"
+          gcloud container clusters get-credentials "${{ secrets.GKE_GPU_CLUSTER_RASA_CI_CD }}" --zone "$GKE_ZONE" --project "${{ secrets.GKE_SA_RASA_CI_GPU_PROJECT_RASA_CI_CD }}"
 
       - name: Remove Github Runner
         run: kubectl -n github-runner delete deployments github-runner-${GITHUB_RUN_ID} --grace-period=30
diff --git a/.github/workflows/continous-integration.yml b/.github/workflows/continous-integration.yml
@@ -30,6 +30,7 @@ env:
   BOTO_CONFIG: /dev/null
   IS_TAG_BUILD: ${{ startsWith(github.event.ref, 'refs/tags') }}
   DOCKERHUB_USERNAME: tmbo
+  DEFAULT_PYTHON_VERSION: '3.7'
 
 jobs:
   changes:
@@ -57,11 +58,11 @@ jobs:
       if: needs.changes.outputs.docs == 'true'
       uses: actions/checkout@v2
 
-    - name: Set up Python 3.7 🐍
+    - name: Set up Python ${{ env.DEFAULT_PYTHON_VERSION }} 🐍
       if: needs.changes.outputs.docs == 'true'
       uses: actions/setup-python@v1
       with:
-        python-version: 3.7
+        python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
 
     - name: Set up Node 12.x 🦙
       if: needs.changes.outputs.docs == 'true'
@@ -86,8 +87,8 @@ jobs:
       uses: actions/cache@v2
       with:
         path: ~/.cache/pypoetry/virtualenvs
-        key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-3.7-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }}
-        restore-keys: ${{ runner.os }}-poetry-3.7
+        key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ env.DEFAULT_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }}
+        restore-keys: ${{ runner.os }}-poetry-${{ env.DEFAULT_PYTHON_VERSION }}
 
     - name: Load Yarn Cached Packages ⬇
       if: needs.changes.outputs.docs == 'true'
@@ -124,11 +125,11 @@ jobs:
       if: needs.changes.outputs.backend == 'true'
       uses: actions/checkout@v2
 
-    - name: Set up Python 3.7 🐍
+    - name: Set up Python ${{ env.DEFAULT_PYTHON_VERSION }} 🐍
       if: needs.changes.outputs.backend == 'true'
       uses: actions/setup-python@v1
       with:
-        python-version: 3.7
+        python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
 
     - name: Read Poetry Version 🔢
       if: needs.changes.outputs.backend == 'true'
@@ -147,8 +148,8 @@ jobs:
       uses: actions/cache@v2
       with:
         path: ~/.cache/pypoetry/virtualenvs
-        key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-3.7-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }}
-        restore-keys: ${{ runner.os }}-poetry-3.7
+        key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ env.DEFAULT_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }}
+        restore-keys: ${{ runner.os }}-poetry-${{ env.DEFAULT_PYTHON_VERSION }}
 
     - name: Install Dependencies 📦
       if: needs.changes.outputs.backend == 'true'
@@ -285,6 +286,137 @@ jobs:
         COVERALLS_SERVICE_NAME: github
       run: poetry run coveralls
 
+  integration_test:
+    name: Run Integration Tests
+    runs-on: ubuntu-latest
+    needs: [changes]
+    env:
+      REDIS_HOST: localhost
+      REDIS_PORT: 6379
+      POSTGRES_HOST: localhost
+      POSTGRES_PORT: 5432
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      RABBITMQ_HOST: localhost
+      RABBITMQ_PORT: 5672
+      RABBITMQ_USER: guest
+      RABBITMQ_PASSWORD: guest
+
+    services:
+      redis:
+        image: redis:6
+        # Set health checks to wait until redis has started
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # FIXME: cannot use ${{ env.REDIS_PORT }} here
+          # mapping container ports to the host
+          - 6379:6379
+
+      postgres:
+        image: postgres:13
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        env:
+          # postgres image requires password to be set
+          POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }}
+        ports:
+          # FIXME: cannot use ${{ env.POSTGRES_PORT }} here
+          # mapping container ports to the host
+          - 5432:5432
+
+      rabbitmq:
+        # see https://github.com/docker-library/healthcheck/blob/master/rabbitmq/docker-healthcheck
+        image: healthcheck/rabbitmq
+        ports:
+          - 5672:5672
+
+    steps:
+    - name: Checkout git repository 🕝
+      if: needs.changes.outputs.backend == 'true'
+      uses: actions/checkout@v2
+
+    - name: Set up Python ${{ env.DEFAULT_PYTHON_VERSION }} 🐍
+      if: needs.changes.outputs.backend == 'true'
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
+
+    - name: Read Poetry Version 🔢
+      if: needs.changes.outputs.backend == 'true'
+      run: |
+        echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV
+      shell: bash
+
+    - name: Install poetry 🦄
+      if: needs.changes.outputs.backend == 'true'
+      uses: Gr1N/setup-poetry@v4
+      with:
+        poetry-version: ${{ env.POETRY_VERSION }}
+
+    - name: Load Poetry Cached Libraries ⬇
+      id: cache-poetry
+      if: needs.changes.outputs.backend == 'true'
+      uses: actions/cache@v2
+      with:
+        path: .venv
+        key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ env.DEFAULT_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}-venv-${{ secrets.POETRY_CACHE_VERSION }}
+
+    - name: Clear Poetry cache
+      if: steps.cache-poetry.outputs.cache-hit == 'true' && needs.changes.outputs.backend == 'true' && contains(github.event.pull_request.labels.*.name, 'tools:clear-poetry-cache-unit-tests')
+      run: rm -r .venv
+
+      # Poetry >= 1.1.0b uses virtualenv to create a virtual environment.
+      # The virtualenv simply doesn't work on Windows with our setup,
+      # that's why we use venv to create virtual environment
+    - name: Create virtual environment
+      if: (steps.cache-poetry.outputs.cache-hit != 'true' || contains(github.event.pull_request.labels.*.name, 'tools:clear-poetry-cache-unit-tests')) && needs.changes.outputs.backend == 'true'
+      run: python -m venv create .venv
+
+    - name: Set up virtual environment
+      if: needs.changes.outputs.backend == 'true'
+      # Poetry on Windows cannot pick up the virtual environments directory properly,
+      # and it creates a new one every time the pipeline runs.
+      # This step solves this problem — it tells poetry to always use `.venv` directory inside
+      # the project itself, which also makes it easier for us to determine the correct directory
+      # that needs to be cached.
+      run: poetry config virtualenvs.in-project true
+
+    - name: Install Dependencies (Linux) 📦
+      if: needs.changes.outputs.backend == 'true'
+      run: |
+        sudo apt-get -y install libpq-dev
+        make install-full | tee .output
+        if grep 'The lock file is not up to date' .output; then exit 1; fi
+        make prepare-tests-ubuntu
+
+    - name: Test Code with Services 🩺
+      if: needs.changes.outputs.backend == 'true'
+      env:
+        JOBS: 2
+        INTEGRATION_TEST_PYTEST_MARKERS: '"not sequential"'
+        PYTHONIOENCODING: "utf-8"
+      run: |
+        make test-integration
+
+    # these integration tests need to be ran in a sequential fashion,
+    # due to environment constraints, so we're running them in a single process.
+    - name: Test Code with Services 🩺 (sequential)
+      if: needs.changes.outputs.backend == 'true'
+      env:
+        JOBS: 1
+        INTEGRATION_TEST_PYTEST_MARKERS: "sequential"
+        PYTHONIOENCODING: "utf-8"
+      run: |
+        make test-integration
+
   build_docker_base_images_and_set_env:
     name: Build Docker base images and setup environment
     runs-on: ubuntu-20.04
@@ -333,7 +465,7 @@ jobs:
         # Fetch branch history
         git fetch --prune --unshallow
         LATEST_TAGGED_NON_ALPHA_RASA_VERSION=$(git tag | sort -r -V | grep -E "^[0-9.]+$" | head -n1)
-        CURRENT_TAG=${GITHUB_TAG/refs\/tags\//}
+        CURRENT_TAG=${GITHUB_REF#refs/tags/}
         # Return 'true' if tag version is equal or higher than the latest tagged Rasa version
         IS_NEWEST_VERSION=$((printf '%s\n%s\n' "${LATEST_TAGGED_NON_ALPHA_RASA_VERSION}" "$CURRENT_TAG" \
           | sort -V -C && echo true || echo false) || true)
@@ -529,7 +661,7 @@ jobs:
             RELEASE_TAG="${IMAGE_TAG}-${{ matrix.image }}"
           fi
 
-          LATEST_TAG=$(echo $RELEASE_TAG | sed 's/'$GITHUB_TAG'/latest/g')
+          LATEST_TAG=$(echo $RELEASE_TAG | sed 's/'$IMAGE_TAG'/latest/g')
 
           docker tag rasa/rasa:${RELEASE_TAG} rasa/rasa:${LATEST_TAG}
           docker push rasa/rasa:${LATEST_TAG}

diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
@@ -64,7 +64,7 @@ jobs:
       run: make install install-docs
 
     - name: Pre-build Docs 🧶
-      run: cd $DOCS_FOLDER && poetry run yarn pre-build
+      run: make prepare-docs
 
     - name: Build & Publish Docs 🏃‍♀️
       env:

diff --git a/.github/workflows/security-scans.yml b/.github/workflows/security-scans.yml
@@ -41,7 +41,7 @@ jobs:
         fetch-depth: '0'
 
     - name: Gitleaks - detecting hardcoded secrets
-      uses: zricethezav/gitleaks-action@v1.1.4
+      uses: zricethezav/gitleaks-action@v1.3.0
 
   bandit:
     name: Detect python security issues