Merge branch 'main' into python-engine-io-fix

Dazu-io · Apr 8, 2021 · d58e3a9 · d58e3a9
2 parents a6ede26 + 7625fca
commit d58e3a9
Show file tree

Hide file tree

Showing 140 changed files with 3,613 additions and 542 deletions.
diff --git a/.github/workflows/ci-model-regression-on-schedule.yml b/.github/workflows/ci-model-regression-on-schedule.yml
@@ -171,6 +171,7 @@ jobs:
         env:
           TFHUB_CACHE_DIR: ~/.tfhub_cache/
           OMP_NUM_THREADS: 1
+          TF_DETERMINISTIC_OPS: 1 # Run GPUs in a deterministic mode
         run: |-
           poetry run rasa --version
 
@@ -220,6 +221,56 @@ jobs:
           name: report.json
           path: ./report.json
 
+      - name: Pick an assignee randomly
+        if: failure()
+        id: pick_assignee
+        run: |
+          ASSIGNEES_LIST=("dakshvar22" "Ghostvv" "JEM-Mosig" "tttthomasssss" "samsucik" "koernerfelicia" "kedz" "aeshky")
+          # Randomly select an item from assignees list
+          ASSIGNEE=${ASSIGNEES_LIST[ $(( RANDOM % ${#ASSIGNEES_LIST[@]} )) ]}
+          echo "::set-output name=assignee::$ASSIGNEE"
+
+      - name: Open GitHub Issue 📬
+        if: failure()
+        uses: rishabhgupta/git-action-issue@v2
+        id: open_issue
+        with:
+            token: ${{ secrets.GITHUB_TOKEN }}
+            title: "Scheduled Model Regression Test Failed"
+            body: "*This PR is automatically created by the Scheduled Model Regression Test workflow. Checkout the Github Action Run [here](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}).* <br> --- <br> **Description of Problem:** <br> Scheduled Model Regression Test failed. <br> **Configuration**: ${{ matrix.config }} <br> **Dataset**: ${{ matrix.dataset}}"
+            assignees: ${{ steps.pick_assignee.outputs.assignee }}
+
+      - name: Add Label to the Issue
+        if: failure()
+        id: add_label
+        env:
+          LABEL: "type:bug :bug:"
+        run: |
+          # Get issue number
+          ISSUE_NUMBER=$(echo '${{ steps.open_issue.outputs.issue }}' | jq -r '.number')
+          echo "::set-output name=issue_number::$ISSUE_NUMBER"
+
+          # Add label to the issue
+          curl \
+            -X POST \
+            -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            -H "Accept: application/vnd.github.v3+json" \
+            https://api.github.com/repos/$GITHUB_REPOSITORY/issues/${ISSUE_NUMBER}/labels \
+            -d '{"labels":["${{ env.LABEL }}"]}'
+
+      - name: Assign Issue to Project Column
+        if: failure()
+        run: |
+          ISSUE_ID=$(echo '${{ steps.open_issue.outputs.issue }}' | jq -r '.id')
+
+          # Add this issue to a project column
+          curl \
+            -X POST \
+            -s -H "Authorization: token ${{ secrets.RASABOT_ASSIGN_ISSUE_RESEARCH_BOARD_TOKEN }}" \
+            -H 'Accept: application/vnd.github.inertia-preview+json' \
+            -d "{\"content_type\": \"Issue\", \"content_id\": $ISSUE_ID}" \
+            "https://api.github.com/projects/columns/${{ secrets.RESEARCH_BOARD_INBOX_COLUMN_ID }}/cards"
+
       - name: Notify Slack of Failure 😱
         if: failure()
         uses: 8398a7/action-slack@v3
@@ -228,12 +279,11 @@ jobs:
           fields: workflow,job,commit,repo,ref,author,took
           custom_payload: |
             {
-              text: "Model regression tests CI",
               attachments: [{
                 fallback: 'fallback',
                 color: '${{ job.status }}' === 'success' ? 'good' : '${{ job.status }}' === 'failure' ? 'danger' : 'warning',
                 title: `${process.env.AS_WORKFLOW}`,
-                text: 'Scheduled model regression test failed',
+                text: 'Scheduled model regression test failed :no_entry:️',
                 fields: [{
                   title: 'Configuration',
                   value: '${{ matrix.config }}',
@@ -244,10 +294,158 @@ jobs:
                   value: '${{ matrix.dataset }}',
                   short: false
                 },
+                {
+                  title: 'GitHub Issue',
+                  value: `https://github.com/${{ github.repository }}/issues/${{ steps.add_label.outputs.issue_number }}`,
+                  short: false
+                },
+                {
+                  title: 'GitHub Action',
+                  value: `https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}`,
+                  short: false
+                }],
+                actions: [{
+                }]
+              }]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_CI_MODEL_REGRESSION_TEST }}
+
+  analyse_performance:
+    name: Analyse Performance
+    runs-on: ubuntu-latest
+    if: always()
+    needs:
+    - deploy_runner_gpu
+    - model_regression_test_gpu
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Download an artifact with the results for the schedule version
+        run: |
+          # Get ID of the schedule workflow
+          SCHEDULE_ID=$(curl -X GET -s -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' -H "Accept: application/vnd.github.v3+json" \
+            "https://api.github.com/repos/${{ github.repository }}/actions/workflows" \
+            | jq -r  '.workflows[] | select(.name == "${{ github.workflow }}") | select(.path | test("schedule")) | .id')
+
+          ARTIFACT_URL=$(curl -s -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}'  -H "Accept: application/vnd.github.v3+json" \
+            "https://api.github.com/repos/${{ github.repository }}/actions/workflows/${SCHEDULE_ID}/runs?event=schedule&status=completed&branch=main&per_page=1" | jq -r .workflow_runs[0].artifacts_url)
+
+          DOWNLOAD_URL=$(curl -s -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' -H "Accept: application/vnd.github.v3+json" "${ARTIFACT_URL}" \
+            | jq -r '.artifacts[] | select(.name="report.json") | .archive_download_url')
+
+          # Download the artifact
+          curl -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' -LJO  -H "Accept: application/vnd.github.v3+json" $DOWNLOAD_URL
+
+          # Unzip and change name
+          unzip report.json.zip && mv report.json report_main.json
+
+      - name: Download the report
+        uses: actions/download-artifact@v2
+        with:
+          name: report.json
+
+      - name: Download gomplate
+        run: |-
+          sudo curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gomplate/releases/download/v3.6.0/gomplate_linux-amd64
+          sudo chmod +x /usr/local/bin/gomplate
+
+      - name: Analyse Performance 🔍
+        id: performance
+        run: |
+          OUTPUT="$(gomplate -d data=report.json -d results_main=report_main.json -f .github/templates/model_regression_test_results.tmpl)"
+
+          OUTPUT="${OUTPUT//$'\n'/'%0A'}"
+          OUTPUT="${OUTPUT//$'\r'/'%0D'}"
+          echo "::set-output name=report_description::${OUTPUT}"
+
+          # Remove all (-0.00) from OUTPUT.
+          # We don't want to count (-0.00) as a performance drop
+          OUTPUT=${OUTPUT//(-0.00)/}
+
+          # Any (negatives) found means there is a performance drops.
+          if [[ "${OUTPUT}" =~ \(-[0-9.]+\) ]]; then
+            echo "Some test performance scores decreased."
+            echo "::set-output name=is_dropped::true"
+          else
+            echo "No test performance drops."
+            echo "::set-output name=is_dropped::false"
+          fi
+
+      - name: Pick an assignee randomly
+        if: steps.performance.outputs.is_dropped == 'true'
+        id: pick_assignee
+        run: |
+          ASSIGNEES_LIST=("dakshvar22" "Ghostvv" "JEM-Mosig" "tttthomasssss" "samsucik" "koernerfelicia" "kedz" "aeshky")
+          # Randomly select an item from assignees list
+          ASSIGNEE=${ASSIGNEES_LIST[ $(( RANDOM % ${#ASSIGNEES_LIST[@]} )) ]}
+          echo "::set-output name=assignee::$ASSIGNEE"
+
+      - name: Open GitHub Issue 📬
+        if: steps.performance.outputs.is_dropped == 'true'
+        uses: rishabhgupta/git-action-issue@v2
+        id: open_issue
+        with:
+            token: ${{ secrets.GITHUB_TOKEN }}
+            title: "Scheduled Model Regression Test Performance Drops"
+            body: "*This PR is automatically created by the Scheduled Model Regression Test workflow. Checkout the Github Action Run [here](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}).* <br> --- <br> **Description of Problem:** <br>  Some test performance scores **decreased**. Please look at the following table for more details. <br> ${{ steps.performance.outputs.report_description }}"
+            assignees: ${{ steps.pick_assignee.outputs.assignee }}
+
+      - name: Add Label to the Issue
+        if: steps.performance.outputs.is_dropped == 'true'
+        id: add_label
+        env:
+          LABEL: "type:bug :bug:"
+        run: |
+          # Get issue number
+          ISSUE_NUMBER=$(echo '${{ steps.open_issue.outputs.issue }}' | jq -r '.number')
+          echo "::set-output name=issue_number::$ISSUE_NUMBER"
+
+          # Add label to the issue
+          curl \
+            -X POST \
+            -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            -H "Accept: application/vnd.github.v3+json" \
+            https://api.github.com/repos/$GITHUB_REPOSITORY/issues/${ISSUE_NUMBER}/labels \
+            -d '{"labels":["${{ env.LABEL }}"]}'
+
+      - name: Assign Issue to Project Column
+        if: steps.performance.outputs.is_dropped == 'true'
+        run: |
+          ISSUE_ID=$(echo '${{ steps.open_issue.outputs.issue }}' | jq -r '.id')
+
+          # Add this issue to a project column
+          curl \
+            -X POST \
+            -s -H "Authorization: token ${{ secrets.RASABOT_ASSIGN_ISSUE_RESEARCH_BOARD_TOKEN }}" \
+            -H 'Accept: application/vnd.github.inertia-preview+json' \
+            -d "{\"content_type\": \"Issue\", \"content_id\": $ISSUE_ID}" \
+            "https://api.github.com/projects/columns/${{ secrets.RESEARCH_BOARD_INBOX_COLUMN_ID }}/cards"
+
+      - name: Notify Slack when Performance Drops 💬
+        if: steps.performance.outputs.is_dropped == 'true'
+        uses: 8398a7/action-slack@v3
+        with:
+          status: custom
+          fields: workflow,job,commit,repo,ref,author,took
+          custom_payload: |
+            {
+              attachments: [{
+                fallback: 'fallback',
+                color: 'danger',
+                title: `${process.env.AS_WORKFLOW}`,
+                text: 'Scheduled model regression test performance drops :chart_with_downwards_trend:',
+                fields: [{
+                  title: 'GitHub Issue',
+                  value: `https://github.com/${{ github.repository }}/issues/${{ steps.add_label.outputs.issue_number }}`,
+                  short: false
+                },
                 {
                   title: 'GitHub Action',
                   value: `https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}`,
-                  short: true
+                  short: false
                 }],
                 actions: [{
                 }]

diff --git a/.github/workflows/continous-integration.yml b/.github/workflows/continous-integration.yml
@@ -72,10 +72,16 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           checkName: Test Documentation
-          ref: ${{ github.head_ref || github.sha }}
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
           timeoutSeconds: ${{ env.WAIT_TIMEOUT_SECS }}
           intervalSeconds: ${{ env.WAIT_INTERVAL_SECS }}
 
+      - name: Fail the step if the doc tests run could not be found
+        if: ${{ steps.wait-for-doc-tests.outputs.conclusion == 'timed_out' }}
+        run: |
+          echo "Could not find the doc tests run."
+          exit 1
+
 
   quality:
     name: Code Quality
@@ -168,6 +174,7 @@ jobs:
         - test-nlu-predictors
         - test-full-model-training
         - test-other-unit-tests
+        - test-performance
         os: [ ubuntu-latest, windows-latest ]
         python-version: [ 3.6, 3.7, 3.8 ]
 

diff --git a/CHANGELOG.mdx b/CHANGELOG.mdx
@@ -16,6 +16,13 @@ https://github.com/RasaHQ/rasa/tree/main/changelog/ . -->
 
 <!-- TOWNCRIER -->
 
+## [2.4.3] - 2021-03-26
+
+
+### Bugfixes
+- [#8114](https://github.com/rasahq/rasa/issues/8114): Fixes bug where stories could not be retrieved if entities had no start or end.
+
+
 ## [2.4.2] - 2021-03-25
 
 

diff --git a/CODEOWNERS b/CODEOWNERS
@@ -1,5 +1,5 @@
 # Code in the `rasa.shared` package is potentially re-used by downstream dependencies
 # such as Rasa X. Hence, changes within this package require double checking.
-/rasa/shared/ @RasaHQ/backend
+/rasa/shared/ @RasaHQ/enable-squad
 /docs/docs/prototype-an-assistant.mdx @ricwo @alwx
 /.github/workflows/ @RasaHQ/infrastructure
diff --git a/Makefile b/Makefile
@@ -102,26 +102,7 @@ lint-security:
 	poetry run bandit -ll -ii -r --config bandit.yml rasa/*
 
 types:
-	# FIXME: working our way towards removing these
-	# see https://github.com/RasaHQ/rasa/pull/6470
-	# the list below is sorted by the number of errors for each error code, in decreasing order
-	MYPYPATH=./stubs poetry run mypy rasa --disable-error-code arg-type \
-	--disable-error-code assignment \
-	--disable-error-code var-annotated \
-	--disable-error-code return-value \
-	--disable-error-code union-attr \
-	--disable-error-code override \
-	--disable-error-code operator \
-	--disable-error-code attr-defined \
-	--disable-error-code index \
-	--disable-error-code misc \
-	--disable-error-code return \
-	--disable-error-code type-var \
-	--disable-error-code has-type \
-	--disable-error-code valid-type \
-	--disable-error-code no-redef \
-	--disable-error-code func-returns-value \
-	--disallow-untyped-calls
+	poetry run mypy rasa
 
 static-checks: lint lint-security types
 
@@ -199,6 +180,9 @@ test-full-model-training: test-marker
 test-other-unit-tests: PYTEST_MARKER=category_other_unit_tests
 test-other-unit-tests: test-marker
 
+test-performance: PYTEST_MARKER=category_performance
+test-performance: test-marker
+
 test-marker: clean
     # OMP_NUM_THREADS can improve overall performance using one thread by process (on tensorflow), avoiding overload
 	OMP_NUM_THREADS=1 poetry run pytest tests -n $(JOBS) --cov rasa -m "$(PYTEST_MARKER)" --ignore $(INTEGRATION_TEST_FOLDER)

diff --git a/changelog/7260.bugfix.md b/changelog/7260.bugfix.md
@@ -0,0 +1 @@
+Fixed the 'loading model' message which was logged twice when using `rasa run`.
diff --git a/changelog/7379.bugfix.md b/changelog/7379.bugfix.md
@@ -0,0 +1 @@
+Change training data validation to only count nlu training examples.
diff --git a/changelog/7827.misc.md b/changelog/7827.misc.md
@@ -0,0 +1 @@
+Add CI checks to detect memory leaks during training.
diff --git a/changelog/8178.bugfix.md b/changelog/8178.bugfix.md
@@ -0,0 +1 @@
+Catch ChannelNotFoundEntity exception coming from the pika broker and raise as ConnectionException.
diff --git a/changelog/8240.misc.md b/changelog/8240.misc.md
@@ -0,0 +1,3 @@
+The [`Component.load`](components.mdx#custom-components) implementation no longer has
+to provide a default value for the `model_dir` parameter and does not need to
+handle potential `None` values.
diff --git a/changelog/8335.enhancement.md b/changelog/8335.enhancement.md
@@ -0,0 +1 @@
+Added partition_by_sender flag to [Kafka Producer](event-brokers.mdx#kafka-event-broker) to optionally associate events with Kafka partition based on sender_id.
diff --git a/changelog/8382.bugfix.md b/changelog/8382.bugfix.md
@@ -0,0 +1,2 @@
+Throw `RasaException` instead of `ValueError` in situations when environment variables 
+specified in YAML cannot be expanded.
diff --git a/data/test_e2ebot/config.yml b/data/test_e2ebot/config.yml
@@ -13,4 +13,5 @@ pipeline:
     epochs: 1
 policies:
 - name: TEDPolicy
+  random_seed: 42
   epochs: 5
diff --git a/data/test_endpoints/event_brokers/kafka_sasl_plaintext_endpoint.yml b/data/test_endpoints/event_brokers/kafka_sasl_plaintext_endpoint.yml
@@ -3,5 +3,6 @@ event_broker:
   security_protocol: SASL_PLAINTEXT
   topic: topic
   url: localhost
+  partition_by_sender: True
   sasl_username: username
   sasl_password: password
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Fixed the 'loading model' message which was logged twice when using `rasa run`.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Change training data validation to only count nlu training examples.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Add CI checks to detect memory leaks during training.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Catch ChannelNotFoundEntity exception coming from the pika broker and raise as ConnectionException.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Added partition_by_sender flag to [Kafka Producer](event-brokers.mdx#kafka-event-broker) to optionally associate events with Kafka partition based on sender_id.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Throw `RasaException` instead of `ValueError` in situations when environment variables
		specified in YAML cannot be expanded.