Skip to content

Commit

Permalink
Merge branch 'main' into misc/merge-2.4.x-main
Browse files Browse the repository at this point in the history
  • Loading branch information
@wochinge
wochinge committed Apr 8, 2021
2 parents 30b8991 + ffa8e75 commit 539fff8
Show file tree
Hide file tree
Showing 249 changed files with 9,212 additions and 3,121 deletions.
17 changes: 17 additions & 0 deletions .github/matchers/flake8-error-matcher.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
{
"problemMatcher": [
{
"owner": "flake8-error",
"severity": "error",
"pattern": [
{
"regexp": "^([^:]+):(\\d+):(\\d+):\\s+([DCFNWE]\\d+\\s+.+)$",
"file": 1,
"line": 2,
"column": 3,
"message": 4
}
]
}
]
}
84 changes: 84 additions & 0 deletions .github/workflows/ci-docs-tests.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
name: Docs Tests
on:
push:
branches:
- main
tags:
- '*'
pull_request:
types: [opened, synchronize, labeled]

env:
DEFAULT_PYTHON_VERSION: '3.7'

jobs:
changes:
name: Check for file changes
runs-on: ubuntu-latest
outputs:
docs: ${{ steps.filter.outputs.docs }}
steps:
- uses: actions/checkout@v2
- uses: RasaHQ/pr-changed-files-filter@c4f7116a04b8a4596313469429e2ad235f59d9c4
id: filter
with:
token: ${{ secrets.GITHUB_TOKEN }}
filters: .github/change_filters.yml

test_documentation:
name: Test Documentation
runs-on: ubuntu-latest
needs: [ changes ]
if: needs.changes.outputs.docs == 'true'

steps:
- name: Checkout git repository 🕝
uses: actions/checkout@v2

- name: Set up Python ${{ env.DEFAULT_PYTHON_VERSION }} 🐍
uses: actions/setup-python@v1
with:
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}

- name: Set up Node 12.x 🦙
uses: actions/setup-node@v1
with:
node-version: '12.x'

- name: Read Poetry Version 🔢
run: |
echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV
shell: bash

- name: Install poetry 🦄
uses: Gr1N/setup-poetry@v4
with:
poetry-version: ${{ env.POETRY_VERSION }}

- name: Load Poetry Cached Libraries ⬇
uses: actions/cache@v2
with:
path: ~/.cache/pypoetry/virtualenvs
key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ env.DEFAULT_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }}
restore-keys: ${{ runner.os }}-poetry-${{ env.DEFAULT_PYTHON_VERSION }}

- name: Load Yarn Cached Packages ⬇
uses: actions/cache@v2
with:
path: docs/node_modules
key: ${{ runner.os }}-yarn-12.x-${{ hashFiles('docs/yarn.lock') }}
restore-keys: ${{ runner.os }}-yarn-12.x

- name: Install Dependencies 📦
run: |
sudo apt-get -y install libpq-dev
make install-full install-docs
- name: Run Swagger 🕵️‍♀️
run: |
npm install -g swagger-cli
swagger-cli validate docs/static/spec/action-server.yml
swagger-cli validate docs/static/spec/rasa.yml
- name: Test Docs 🕸
run: make test-docs
205 changes: 201 additions & 4 deletions .github/workflows/ci-model-regression-on-schedule.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,6 @@ jobs:
poetry install --extras full
make install
poetry run python -m spacy download de_core_news_md
poetry run python -m spacy link --force de_core_news_md de
- name: Validate that GPUs are working
if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
Expand All @@ -172,6 +171,7 @@ jobs:
env:
TFHUB_CACHE_DIR: ~/.tfhub_cache/
OMP_NUM_THREADS: 1
TF_DETERMINISTIC_OPS: 1 # Run GPUs in a deterministic mode
run: |-
poetry run rasa --version
Expand Down Expand Up @@ -221,6 +221,56 @@ jobs:
name: report.json
path: ./report.json

- name: Pick an assignee randomly
if: failure()
id: pick_assignee
run: |
ASSIGNEES_LIST=("dakshvar22" "Ghostvv" "JEM-Mosig" "tttthomasssss" "samsucik" "koernerfelicia" "kedz" "aeshky")
# Randomly select an item from assignees list
ASSIGNEE=${ASSIGNEES_LIST[ $(( RANDOM % ${#ASSIGNEES_LIST[@]} )) ]}
echo "::set-output name=assignee::$ASSIGNEE"
- name: Open GitHub Issue 📬
if: failure()
uses: rishabhgupta/git-action-issue@v2
id: open_issue
with:
token: ${{ secrets.GITHUB_TOKEN }}
title: "Scheduled Model Regression Test Failed"
body: "*This PR is automatically created by the Scheduled Model Regression Test workflow. Checkout the Github Action Run [here](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}).* <br> --- <br> **Description of Problem:** <br> Scheduled Model Regression Test failed. <br> **Configuration**: ${{ matrix.config }} <br> **Dataset**: ${{ matrix.dataset}}"
assignees: ${{ steps.pick_assignee.outputs.assignee }}

- name: Add Label to the Issue
if: failure()
id: add_label
env:
LABEL: "type:bug :bug:"
run: |
# Get issue number
ISSUE_NUMBER=$(echo '${{ steps.open_issue.outputs.issue }}' | jq -r '.number')
echo "::set-output name=issue_number::$ISSUE_NUMBER"
# Add label to the issue
curl \
-X POST \
-H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
-H "Accept: application/vnd.github.v3+json" \
https://api.github.com/repos/$GITHUB_REPOSITORY/issues/${ISSUE_NUMBER}/labels \
-d '{"labels":["${{ env.LABEL }}"]}'
- name: Assign Issue to Project Column
if: failure()
run: |
ISSUE_ID=$(echo '${{ steps.open_issue.outputs.issue }}' | jq -r '.id')
# Add this issue to a project column
curl \
-X POST \
-s -H "Authorization: token ${{ secrets.RASABOT_ASSIGN_ISSUE_RESEARCH_BOARD_TOKEN }}" \
-H 'Accept: application/vnd.github.inertia-preview+json' \
-d "{\"content_type\": \"Issue\", \"content_id\": $ISSUE_ID}" \
"https://api.github.com/projects/columns/${{ secrets.RESEARCH_BOARD_INBOX_COLUMN_ID }}/cards"
- name: Notify Slack of Failure 😱
if: failure()
uses: 8398a7/action-slack@v3
Expand All @@ -229,12 +279,11 @@ jobs:
fields: workflow,job,commit,repo,ref,author,took
custom_payload: |
{
text: "Model regression tests CI",
attachments: [{
fallback: 'fallback',
color: '${{ job.status }}' === 'success' ? 'good' : '${{ job.status }}' === 'failure' ? 'danger' : 'warning',
title: `${process.env.AS_WORKFLOW}`,
text: 'Scheduled model regression test failed',
text: 'Scheduled model regression test failed :no_entry:️',
fields: [{
title: 'Configuration',
value: '${{ matrix.config }}',
Expand All @@ -245,10 +294,158 @@ jobs:
value: '${{ matrix.dataset }}',
short: false
},
{
title: 'GitHub Issue',
value: `https://github.com/${{ github.repository }}/issues/${{ steps.add_label.outputs.issue_number }}`,
short: false
},
{
title: 'GitHub Action',
value: `https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}`,
short: false
}],
actions: [{
}]
}]
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_CI_MODEL_REGRESSION_TEST }}

analyse_performance:
name: Analyse Performance
runs-on: ubuntu-latest
if: always()
needs:
- deploy_runner_gpu
- model_regression_test_gpu

steps:
- name: Checkout
uses: actions/checkout@v2

- name: Download an artifact with the results for the schedule version
run: |
# Get ID of the schedule workflow
SCHEDULE_ID=$(curl -X GET -s -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' -H "Accept: application/vnd.github.v3+json" \
"https://api.github.com/repos/${{ github.repository }}/actions/workflows" \
| jq -r '.workflows[] | select(.name == "${{ github.workflow }}") | select(.path | test("schedule")) | .id')
ARTIFACT_URL=$(curl -s -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' -H "Accept: application/vnd.github.v3+json" \
"https://api.github.com/repos/${{ github.repository }}/actions/workflows/${SCHEDULE_ID}/runs?event=schedule&status=completed&branch=main&per_page=1" | jq -r .workflow_runs[0].artifacts_url)
DOWNLOAD_URL=$(curl -s -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' -H "Accept: application/vnd.github.v3+json" "${ARTIFACT_URL}" \
| jq -r '.artifacts[] | select(.name="report.json") | .archive_download_url')
# Download the artifact
curl -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' -LJO -H "Accept: application/vnd.github.v3+json" $DOWNLOAD_URL
# Unzip and change name
unzip report.json.zip && mv report.json report_main.json
- name: Download the report
uses: actions/download-artifact@v2
with:
name: report.json

- name: Download gomplate
run: |-
sudo curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gomplate/releases/download/v3.6.0/gomplate_linux-amd64
sudo chmod +x /usr/local/bin/gomplate
- name: Analyse Performance 🔍
id: performance
run: |
OUTPUT="$(gomplate -d data=report.json -d results_main=report_main.json -f .github/templates/model_regression_test_results.tmpl)"
OUTPUT="${OUTPUT//$'\n'/'%0A'}"
OUTPUT="${OUTPUT//$'\r'/'%0D'}"
echo "::set-output name=report_description::${OUTPUT}"
# Remove all (-0.00) from OUTPUT.
# We don't want to count (-0.00) as a performance drop
OUTPUT=${OUTPUT//(-0.00)/}
# Any (negatives) found means there is a performance drops.
if [[ "${OUTPUT}" =~ \(-[0-9.]+\) ]]; then
echo "Some test performance scores decreased."
echo "::set-output name=is_dropped::true"
else
echo "No test performance drops."
echo "::set-output name=is_dropped::false"
fi
- name: Pick an assignee randomly
if: steps.performance.outputs.is_dropped == 'true'
id: pick_assignee
run: |
ASSIGNEES_LIST=("dakshvar22" "Ghostvv" "JEM-Mosig" "tttthomasssss" "samsucik" "koernerfelicia" "kedz" "aeshky")
# Randomly select an item from assignees list
ASSIGNEE=${ASSIGNEES_LIST[ $(( RANDOM % ${#ASSIGNEES_LIST[@]} )) ]}
echo "::set-output name=assignee::$ASSIGNEE"
- name: Open GitHub Issue 📬
if: steps.performance.outputs.is_dropped == 'true'
uses: rishabhgupta/git-action-issue@v2
id: open_issue
with:
token: ${{ secrets.GITHUB_TOKEN }}
title: "Scheduled Model Regression Test Performance Drops"
body: "*This PR is automatically created by the Scheduled Model Regression Test workflow. Checkout the Github Action Run [here](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}).* <br> --- <br> **Description of Problem:** <br> Some test performance scores **decreased**. Please look at the following table for more details. <br> ${{ steps.performance.outputs.report_description }}"
assignees: ${{ steps.pick_assignee.outputs.assignee }}

- name: Add Label to the Issue
if: steps.performance.outputs.is_dropped == 'true'
id: add_label
env:
LABEL: "type:bug :bug:"
run: |
# Get issue number
ISSUE_NUMBER=$(echo '${{ steps.open_issue.outputs.issue }}' | jq -r '.number')
echo "::set-output name=issue_number::$ISSUE_NUMBER"
# Add label to the issue
curl \
-X POST \
-H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
-H "Accept: application/vnd.github.v3+json" \
https://api.github.com/repos/$GITHUB_REPOSITORY/issues/${ISSUE_NUMBER}/labels \
-d '{"labels":["${{ env.LABEL }}"]}'
- name: Assign Issue to Project Column
if: steps.performance.outputs.is_dropped == 'true'
run: |
ISSUE_ID=$(echo '${{ steps.open_issue.outputs.issue }}' | jq -r '.id')
# Add this issue to a project column
curl \
-X POST \
-s -H "Authorization: token ${{ secrets.RASABOT_ASSIGN_ISSUE_RESEARCH_BOARD_TOKEN }}" \
-H 'Accept: application/vnd.github.inertia-preview+json' \
-d "{\"content_type\": \"Issue\", \"content_id\": $ISSUE_ID}" \
"https://api.github.com/projects/columns/${{ secrets.RESEARCH_BOARD_INBOX_COLUMN_ID }}/cards"
- name: Notify Slack when Performance Drops 💬
if: steps.performance.outputs.is_dropped == 'true'
uses: 8398a7/action-slack@v3
with:
status: custom
fields: workflow,job,commit,repo,ref,author,took
custom_payload: |
{
attachments: [{
fallback: 'fallback',
color: 'danger',
title: `${process.env.AS_WORKFLOW}`,
text: 'Scheduled model regression test performance drops :chart_with_downwards_trend:',
fields: [{
title: 'GitHub Issue',
value: `https://github.com/${{ github.repository }}/issues/${{ steps.add_label.outputs.issue_number }}`,
short: false
},
{
title: 'GitHub Action',
value: `https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}`,
short: true
short: false
}],
actions: [{
}]
Expand Down
2 changes: 0 additions & 2 deletions .github/workflows/ci-model-regression.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,6 @@ jobs:
poetry install --extras full
make install
poetry run python -m spacy download de_core_news_md
poetry run python -m spacy link --force de_core_news_md de
- name: Validate that GPUs are working
if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
Expand Down Expand Up @@ -400,7 +399,6 @@ jobs:
poetry install --extras full
make install
poetry run python -m spacy download de_core_news_md
poetry run python -m spacy link --force de_core_news_md de
- name: Run test
id: run_test
Expand Down
Loading

0 comments on commit 539fff8

Please sign in to comment.