Merge branch 'main' into merge/2.4.2-to-main

Dazu-io · Mar 25, 2021 · 1168b59 · 1168b59
2 parents 396aadf + 4b8f745
commit 1168b59
Show file tree

Hide file tree

Showing 205 changed files with 5,410 additions and 2,548 deletions.
diff --git a/.github/workflows/ci-docs-tests.yml b/.github/workflows/ci-docs-tests.yml
@@ -0,0 +1,84 @@
+name: Docs Tests
+on:
+  push:
+    branches:
+    - main
+    tags:
+    - '*'
+  pull_request:
+    types: [opened, synchronize, labeled]
+
+env:
+  DEFAULT_PYTHON_VERSION: '3.7'
+
+jobs:
+  changes:
+    name: Check for file changes
+    runs-on: ubuntu-latest
+    outputs:
+      docs: ${{ steps.filter.outputs.docs }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: RasaHQ/pr-changed-files-filter@c4f7116a04b8a4596313469429e2ad235f59d9c4
+        id: filter
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          filters: .github/change_filters.yml
+
+  test_documentation:
+    name: Test Documentation
+    runs-on: ubuntu-latest
+    needs: [ changes ]
+    if: needs.changes.outputs.docs == 'true'
+
+    steps:
+    - name: Checkout git repository 🕝
+      uses: actions/checkout@v2
+
+    - name: Set up Python ${{ env.DEFAULT_PYTHON_VERSION }} 🐍
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
+
+    - name: Set up Node 12.x 🦙
+      uses: actions/setup-node@v1
+      with:
+        node-version: '12.x'
+
+    - name: Read Poetry Version 🔢
+      run: |
+        echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV
+      shell: bash
+
+    - name: Install poetry 🦄
+      uses: Gr1N/setup-poetry@v4
+      with:
+        poetry-version: ${{ env.POETRY_VERSION }}
+
+    - name: Load Poetry Cached Libraries ⬇
+      uses: actions/cache@v2
+      with:
+        path: ~/.cache/pypoetry/virtualenvs
+        key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ env.DEFAULT_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }}
+        restore-keys: ${{ runner.os }}-poetry-${{ env.DEFAULT_PYTHON_VERSION }}
+
+    - name: Load Yarn Cached Packages ⬇
+      uses: actions/cache@v2
+      with:
+        path: docs/node_modules
+        key: ${{ runner.os }}-yarn-12.x-${{ hashFiles('docs/yarn.lock') }}
+        restore-keys: ${{ runner.os }}-yarn-12.x
+
+    - name: Install Dependencies 📦
+      run: |
+        sudo apt-get -y install libpq-dev
+        make install-full install-docs
+
+    - name: Run Swagger 🕵️‍♀️
+      run: |
+        npm install -g swagger-cli
+        swagger-cli validate docs/static/spec/action-server.yml
+        swagger-cli validate docs/static/spec/rasa.yml
+
+    - name: Test Docs 🕸
+      run: make test-docs
diff --git a/.github/workflows/continous-integration.yml b/.github/workflows/continous-integration.yml
@@ -28,10 +28,16 @@ env:
   # needed to fix issues with boto during testing:
   # https://github.com/travis-ci/travis-ci/issues/7940
   BOTO_CONFIG: /dev/null
+
   IS_TAG_BUILD: ${{ startsWith(github.event.ref, 'refs/tags') }}
   DOCKERHUB_USERNAME: tmbo
   DEFAULT_PYTHON_VERSION: '3.7'
 
+  # for wait_for_xx jobs
+  WAIT_TIMEOUT_SECS: 3000
+  WAIT_INTERVAL_SECS: 60
+
+
 jobs:
   changes:
     name: Check for file changes
@@ -48,72 +54,28 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           filters: .github/change_filters.yml
 
-  documentation:
-    name: Test Documentation
+
+  wait_for_docs_tests:
+    # Looks for doc test workflows and waits for it to complete successfully
+    # Runs on tag pushes and pushes to main exclusively, as it is a dependency of release jobs
+    name: Wait for docs tests
     runs-on: ubuntu-latest
-    needs: [changes]
+    needs: [ changes ]
 
     steps:
-    - name: Checkout git repository 🕝
-      if: needs.changes.outputs.docs == 'true'
-      uses: actions/checkout@v2
+      - name: Checkout git repository 🕝
+        uses: actions/checkout@v2
 
-    - name: Set up Python ${{ env.DEFAULT_PYTHON_VERSION }} 🐍
-      if: needs.changes.outputs.docs == 'true'
-      uses: actions/setup-python@v1
-      with:
-        python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
-
-    - name: Set up Node 12.x 🦙
-      if: needs.changes.outputs.docs == 'true'
-      uses: actions/setup-node@v1
-      with:
-        node-version: '12.x'
-
-    - name: Read Poetry Version 🔢
-      if: needs.changes.outputs.docs == 'true'
-      run: |
-        echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV
-      shell: bash
-
-    - name: Install poetry 🦄
-      if: needs.changes.outputs.docs == 'true'
-      uses: Gr1N/setup-poetry@v4
-      with:
-        poetry-version: ${{ env.POETRY_VERSION }}
-
-    - name: Load Poetry Cached Libraries ⬇
-      if: needs.changes.outputs.docs == 'true'
-      uses: actions/cache@v2
-      with:
-        path: ~/.cache/pypoetry/virtualenvs
-        key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ env.DEFAULT_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }}
-        restore-keys: ${{ runner.os }}-poetry-${{ env.DEFAULT_PYTHON_VERSION }}
-
-    - name: Load Yarn Cached Packages ⬇
-      if: needs.changes.outputs.docs == 'true'
-      uses: actions/cache@v2
-      with:
-        path: docs/node_modules
-        key: ${{ runner.os }}-yarn-12.x-${{ hashFiles('docs/yarn.lock') }}
-        restore-keys: ${{ runner.os }}-yarn-12.x
-
-    - name: Install Dependencies 📦
-      if: needs.changes.outputs.docs == 'true'
-      run: |
-        sudo apt-get -y install libpq-dev
-        make install-full install-docs
-
-    - name: Run Swagger 🕵️‍♀️
-      if: needs.changes.outputs.docs == 'true'
-      run: |
-        npm install -g swagger-cli
-        swagger-cli validate docs/static/spec/action-server.yml
-        swagger-cli validate docs/static/spec/rasa.yml
+      - name: Wait for doc tests
+        uses: fountainhead/action-wait-for-check@4699210ccc66e2a13260803fadbb77085421b891
+        id: wait-for-doc-tests
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          checkName: Test Documentation
+          ref: ${{ github.head_ref || github.sha }}
+          timeoutSeconds: ${{ env.WAIT_TIMEOUT_SECS }}
+          intervalSeconds: ${{ env.WAIT_INTERVAL_SECS }}
 
-    - name: Test Docs 🕸
-      if: needs.changes.outputs.docs == 'true'
-      run: make test-docs
 
   quality:
     name: Code Quality
@@ -683,7 +645,7 @@ jobs:
 
     # deploy will only be run when there is a tag available
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') && github.repository == 'RasaHQ/rasa'
-    needs: [quality, test, documentation, docker]  # only run after all other stages succeeded
+    needs: [quality, test, wait_for_docs_tests, docker]  # only run after all other stages succeeded
 
     steps:
     - name: Checkout git repository 🕝

diff --git a/Makefile b/Makefile
@@ -105,7 +105,7 @@ types:
 	# FIXME: working our way towards removing these
 	# see https://github.com/RasaHQ/rasa/pull/6470
 	# the list below is sorted by the number of errors for each error code, in decreasing order
-	poetry run mypy rasa --disable-error-code arg-type \
+	MYPYPATH=./stubs poetry run mypy rasa --disable-error-code arg-type \
 	--disable-error-code assignment \
 	--disable-error-code var-annotated \
 	--disable-error-code return-value \
@@ -116,23 +116,19 @@ types:
 	--disable-error-code index \
 	--disable-error-code misc \
 	--disable-error-code return \
-	--disable-error-code call-arg \
 	--disable-error-code type-var \
-	--disable-error-code list-item \
 	--disable-error-code has-type \
 	--disable-error-code valid-type \
-	--disable-error-code dict-item \
 	--disable-error-code no-redef \
-	--disable-error-code func-returns-value
+	--disable-error-code func-returns-value \
+	--disallow-untyped-calls
 
 static-checks: lint lint-security types
 
 prepare-spacy:
 	poetry install -E spacy
 	poetry run python -m spacy download en_core_web_md
 	poetry run python -m spacy download de_core_news_sm
-	poetry run python -m spacy link en_core_web_md en --force
-	poetry run python -m spacy link de_core_news_sm de --force
 
 prepare-mitie:
 	wget --progress=dot:giga -N -P data/ https://github.com/mit-nlp/MITIE/releases/download/v0.4/MITIE-models-v0.2.tar.bz2

diff --git a/changelog/4280.improvement.md b/changelog/4280.improvement.md
@@ -0,0 +1 @@
+Improved CLI startup time.
diff --git a/changelog/4596.improvement.md b/changelog/4596.improvement.md
@@ -0,0 +1,3 @@
+Add `augmentation` and `num_threads` arguments to API `POST /model/train`
+
+Fix boolean casting issue for `force_training` and `save_to_default_model_directory` arguments
diff --git a/changelog/6511.misc.md b/changelog/6511.misc.md
@@ -0,0 +1,3 @@
+API endpoint `POST /model/test/intents` now returns HTTP 409 status
+code in case it cannot find the NLU model directory, instead of an
+HTTP 500 status.
diff --git a/changelog/7450.bugfix.md b/changelog/7450.bugfix.md
@@ -0,0 +1,2 @@
+Rule tracker states no longer include the initial value of slots.
+Rules now only require slot values when explicitly stated in the rule.
diff --git a/changelog/7477.enhancement.md b/changelog/7477.enhancement.md
@@ -0,0 +1 @@
+Add minimum compatible version to --version command
diff --git a/changelog/7640.bugfix.md b/changelog/7640.bugfix.md
@@ -0,0 +1,2 @@
+`rasa test`, `rasa test core` and `rasa test nlu` no longer show temporary paths
+in case there are issues in the test files.
diff --git a/changelog/7640.misc.md b/changelog/7640.misc.md
@@ -0,0 +1,15 @@
+The [`TrainingDataImporter` interface](training-data-importers.mdx) now permits the
+implementation of a method `get_conversation_tests` which specifically retrieves
+[end-to-end conversation tests](testing-your-assistant.mdx#writing-test-stories). 
+Its signature looks as follows:
+
+```python
+
+ async def get_conversation_tests(self) -> StoryGraph:
+        """Retrieves end-to-end conversation stories for testing.
+
+        Returns:
+            `StoryGraph` containing all loaded stories.
+        """
+        pass
+```
diff --git a/changelog/7660.improvement.md b/changelog/7660.improvement.md
@@ -0,0 +1,2 @@
+Updated warning for unexpected slot events during prediction time to Rasa Open Source
+2.0 YAML training data format.
diff --git a/changelog/7701.improvement.md b/changelog/7701.improvement.md
@@ -0,0 +1,10 @@
+Hide dialogue turns predicted by `RulePolicy` in the tracker states
+for ML-only policies like `TEDPolicy`
+if those dialogue turns only appear as rules in the training data and do not appear in stories.
+
+Add `set_shared_policy_states(...)` method to all policies.
+This method sets `_rule_only_data` dict with keys:
+- `rule_only_slots`: Slot names, which only occur in rules but not in stories.
+- `rule_only_loops`: Loop names, which only occur in rules but not in stories.
+
+This information is needed for correct featurization to hide dialogue turns that appear only in rules.
diff --git a/changelog/7869.feature.md b/changelog/7869.feature.md
@@ -0,0 +1,41 @@
+Upgraded Rasa to be compatible with spaCy 3.0. 
+
+This means that we can support more features for more languages but there are also a few changes. 
+
+SpaCy 3.0 deprecated the `spacy link <language model>` command so that means that from now on [the 
+full model name](https://spacy.io/models) needs to be used in the `config.yml` file. 
+
+**Before** 
+
+Before you could run `spacy link en en_core_web_md` and then we would be able
+to pick up the correct model from the `language` parameter. 
+
+```yaml
+language: en
+
+pipeline:
+   - name: SpacyNLP
+```
+
+**Now** 
+
+This behavior will be deprecated and instead you'll want to be explicit in `config.yml`.
+
+```yaml
+language: en
+
+pipeline:
+   - name: SpacyNLP
+     model: en_core_web_md
+```
+
+**Fallback** 
+
+To make the transition easier, Rasa will try to fall back to a medium spaCy model when-ever 
+a compatible language is configured for the entire pipeline in `config.yml` even if you don't
+specify a `model`. This fallback behavior is temporary and will be deprecated in Rasa 3.0.0.  
+
+We've updated our docs to reflect these changes. All examples now show a direct link to the 
+correct spaCy model. We've also added a warning to the [SpaCyNLP](components.mdx#spacynlp) 
+docs that explains the fallback behavior. 
+
diff --git a/changelog/8056.misc.md b/changelog/8056.misc.md
@@ -0,0 +1 @@
+Remove `tests/import_time.py`
diff --git a/changelog/8102.bugfix.md b/changelog/8102.bugfix.md
@@ -0,0 +1 @@
+Fixed a bug in a form where slot mapping doesn't work if the predicted intent name is substring for another intent name.
diff --git a/changelog/8141.misc.md b/changelog/8141.misc.md
@@ -0,0 +1,3 @@
+The following modules were renamed:
+* `rasa.train` -> `rasa.model_training`
+* `rasa.test` -> `rasa.model_testing`
diff --git a/changelog/8141.removal.md b/changelog/8141.removal.md
@@ -0,0 +1,6 @@
+The following import abbreviations were removed:
+* `rasa.core.train`: Please use `rasa.core.train.train` instead.
+* `rasa.core.visualize`: Please use `rasa.core.visualize.visualize` instead.
+* `rasa.nlu.train`: Please use `rasa.nlu.train.train` instead.
+* `rasa.nlu.test`: Please use `rasa.nlu.test.run_evaluation` instead.
+* `rasa.nlu.cross_validate`: Please use `rasa.nlu.test.cross_validate` instead.
diff --git a/changelog/8208.improvement.md b/changelog/8208.improvement.md
@@ -0,0 +1 @@
+Faster reading of YAML NLU training data files.
diff --git a/data/test_config/config_pretrained_embeddings_spacy_de.yml b/data/test_config/config_pretrained_embeddings_spacy_de.yml
@@ -2,6 +2,7 @@ language: "de"
 
 pipeline:
   - name: SpacyNLP
+    model: "de_core_news_sm"
   - name: SpacyTokenizer
   - name: SpacyFeaturizer
   - name: RegexFeaturizer

diff --git a/data/test_config/config_spacy_entity_extractor.yml b/data/test_config/config_spacy_entity_extractor.yml
@@ -1,6 +1,7 @@
 language: en
 pipeline:
     - name: "SpacyNLP"
+      model: "en_core_web_md"
     - name: "SpacyTokenizer"
     - name: "SpacyFeaturizer"
     - name: "RegexFeaturizer"

diff --git a/data/test_dialogues/default.json b/data/test_dialogues/default.json
@@ -9,7 +9,8 @@
       "confidence": null,
       "policy": null,
       "timestamp": 1551952977.4850519,
-      "unpredictable": false
+      "unpredictable": false,
+      "hide_rule_turn": false
     },
     {
       "py/object": "rasa.shared.core.events.UserUttered",
@@ -61,7 +62,8 @@
       "confidence": null,
       "policy": null,
       "timestamp": 1551953040.607782,
-      "unpredictable": false
+      "unpredictable": false,
+      "hide_rule_turn": false
     },
     {
       "py/object": "rasa.shared.core.events.BotUttered",
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		Add `augmentation` and `num_threads` arguments to API `POST /model/train`

		Fix boolean casting issue for `force_training` and `save_to_default_model_directory` arguments
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Rule tracker states no longer include the initial value of slots.
		Rules now only require slot values when explicitly stated in the rule.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Add minimum compatible version to --version command
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		`rasa test`, `rasa test core` and `rasa test nlu` no longer show temporary paths
		in case there are issues in the test files.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Updated warning for unexpected slot events during prediction time to Rasa Open Source
		2.0 YAML training data format.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Fixed a bug in a form where slot mapping doesn't work if the predicted intent name is substring for another intent name.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Faster reading of YAML NLU training data files.