regression_tests #140
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: regression_tests | |
| on: | |
| push: | |
| pull_request: | |
| schedule: | |
| - cron: '0 0 * * *' # every day at midnight | |
| workflow_dispatch: | |
| inputs: | |
| default_only: | |
| description: 'Test default scenarios only' | |
| required: false | |
| type: boolean | |
| commit_id: | |
| description: 'Commit ID' | |
| required: false | |
| default: '' | |
| marker: | |
| description: 'Pytest Marker (defined in pytest.ini)' | |
| required: false | |
| default: '' | |
| type: choice | |
| options: | |
| - '' | |
| - ci | |
| - light | |
| - ha | |
| - ipfix | |
| - bgp | |
| - bmp | |
| - avro | |
| - json | |
| - redis | |
| - signals | |
| - memory_leak | |
| #Global vars | |
| env: | |
| #TODO: avoid duplicity ci/regression_tests | |
| DAEMONS: "pmacctd nfacctd sfacctd uacctd pmbgpd pmbmpd pmtelemetryd" | |
| MARKER: "ci" # Default marker for normal CI execution | |
| COMMIT_ID: ${{ github.sha }} | |
| DEFAULT_ONLY: false | |
| jobs: | |
| ### Step 1: Build Traffic Reproducer Images | |
| traf-repro-docker: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout pmacct | |
| uses: actions/checkout@v4 | |
| with: | |
| path: pmacct | |
| submodules: recursive | |
| - name: Create Traffic Reproducer Docker Images | |
| run: | | |
| sudo apt update | |
| sudo apt install docker | |
| cd pmacct/test-framework | |
| tools/pcap_player/build_docker_image.sh | |
| - name: Check Images and Save as Artifacts | |
| run: | | |
| echo "Checking Images..." | |
| docker images | grep _build | |
| echo | |
| echo "Saving images as artifacts..." | |
| mkdir -p /tmp/docker/ | |
| docker save -o /tmp/docker/traffic_reproducer_docker_images.tar traffic-reproducer:_build | |
| - name: Upload Artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| retention-days: 1 | |
| name: traffic_reproducer_docker_images | |
| path: /tmp/docker | |
| ### Step 2: Retrieve all other necessary images from Docker Hub and store as artifacts | |
| ### (to avoid possible Docker Hub pull limits) | |
| cache-docker-images: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout pmacct | |
| uses: actions/checkout@v4 | |
| with: | |
| path: pmacct | |
| fetch-depth: 0 | |
| fetch-tags: 1 | |
| - name: Overwrite environment variables for the workflow_dispatch event | |
| run: | | |
| if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then | |
| echo "MARKER=${{ inputs.marker }}" >> $GITHUB_ENV | |
| if [ ! -z "${{ inputs.commit_id }}" ]; then | |
| echo "COMMIT_ID=${{ inputs.commit_id }}" >> $GITHUB_ENV | |
| fi | |
| echo "DEFAULT_ONLY=${{ inputs.default_only }}" >> $GITHUB_ENV | |
| fi | |
| - name: Build containers | |
| uses: ./pmacct/.github/actions/build_containers/ | |
| with: | |
| daemons: ${{env.DAEMONS}} | |
| commit-id: ${{env.COMMIT_ID}} | |
| - name: Download images and prepare artifacts | |
| run: | | |
| cd pmacct/test-framework | |
| source settings.conf | |
| docker image pull $ZOOKEEPER_IMG | |
| docker image pull $KAFKA_IMG | |
| docker image pull $SCHEMAREGISTRY_IMG | |
| docker image pull $REDIS_IMG | |
| echo "List Images" | |
| docker images | |
| echo | |
| echo "Saving images as artifacts..." | |
| mkdir -p /tmp/docker/ | |
| PMACCT_IMAGES=$(docker image ls --format '{{.Repository}}:{{.Tag}}' | grep "_build") | |
| docker save -o /tmp/docker/hub_pulled_docker_images.tar $ZOOKEEPER_IMG $KAFKA_IMG $SCHEMAREGISTRY_IMG $REDIS_IMG $PMACCT_IMAGES | |
| - name: Upload Artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| retention-days: 1 | |
| name: hub_pulled_docker_images | |
| path: /tmp/docker | |
| ### Step 3: Collect tests from framework | |
| collect-tests: | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - name: Checkout pmacct | |
| uses: actions/checkout@v4 | |
| with: | |
| path: pmacct | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Install Framework Requirements | |
| run: | | |
| sudo apt update | |
| sudo apt install librdkafka-dev docker | |
| pip install --upgrade pip | |
| pip install -r pmacct/test-framework/requirements.txt | |
| - name: Overwrite environment variables for the workflow_dispatch event | |
| run: | | |
| if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then | |
| echo "MARKER=${{ inputs.marker }}" >> $GITHUB_ENV | |
| if [ ! -z "${{ inputs.commit_id }}" ]; then | |
| echo "COMMIT_ID=${{ inputs.commit_id }}" >> $GITHUB_ENV | |
| fi | |
| echo "DEFAULT_ONLY=${{ inputs.default_only }}" >> $GITHUB_ENV | |
| fi | |
| - name: Collect list of tests matching with the provided marker | |
| id: set-matrix | |
| env: | |
| MARKER: ${{ env.MARKER }} | |
| run: | | |
| cd pmacct/test-framework | |
| dry_run_output=$(sudo env PATH="$PATH" ./runtest.sh --dry * --mark="$MARKER") | |
| MATRIX="{"test": $(echo "$dry_run_output" | grep -oP '(?<=<Module )\d+(?=_test\.py)' | jq -R -s -c 'split("\n")[:-1]')}" | |
| echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT | |
| echo "Collected Tests:" | |
| echo $MATRIX | |
| ### Step 4: Setup Framework and Run Tests | |
| pytest-runtests: | |
| needs: [collect-tests, traf-repro-docker, cache-docker-images] | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.collect-tests.outputs.matrix) }} | |
| steps: | |
| - name: Checkout pmacct | |
| uses: actions/checkout@v4 | |
| with: | |
| path: pmacct | |
| fetch-depth: 0 | |
| fetch-tags: 1 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Install Framework Requirements | |
| run: | | |
| sudo apt update | |
| sudo apt install librdkafka-dev docker | |
| pip install --upgrade pip | |
| pip install -r pmacct/test-framework/requirements.txt | |
| - name: Download Artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: '*_docker_images' | |
| path: /tmp/docker | |
| - name: Import images in the local registry | |
| run: | | |
| docker load -i /tmp/docker/traffic_reproducer_docker_images/traffic_reproducer_docker_images.tar | |
| docker load -i /tmp/docker/hub_pulled_docker_images/hub_pulled_docker_images.tar | |
| echo "List Images" | |
| docker images | grep 'confluentinc\|redis\|traffic\|_build' | |
| - name: Overwrite environment variables for the workflow_dispatch event | |
| run: | | |
| if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then | |
| echo "MARKER=${{ inputs.marker }}" >> $GITHUB_ENV | |
| if [ ! -z "${{ inputs.commit_id }}" ]; then | |
| echo "COMMIT_ID=${{ inputs.commit_id }}" >> $GITHUB_ENV | |
| fi | |
| echo "DEFAULT_ONLY=${{ inputs.default_only }}" >> $GITHUB_ENV | |
| fi | |
| - name: Run the test(s) | |
| env: | |
| DEFAULT_ONLY: ${{ env.DEFAULT_ONLY }} | |
| run: | | |
| cd pmacct/test-framework | |
| if [[ "$DEFAULT_ONLY" == "true" ]]; then | |
| sudo env PATH="$PATH" ./runtest.sh ${{ matrix.test }}:00 | |
| else | |
| sudo env PATH="$PATH" ./runtest.sh ${{ matrix.test }} | |
| fi | |
| - name: Prepare Results Folder for Upload (permissions and folder name) | |
| if: '!cancelled()' # always run this step, unless job manually cancelled | |
| run: | | |
| cd pmacct/test-framework | |
| sudo chown -R 1000:1000 results/ | |
| sudo chmod -R 777 results/ | |
| echo "Adjust results folder name (when : or * is used as part of ./runtest.sh argument...)" | |
| TEST_FOLDER_NAME=$( echo ${{ matrix.test }} | sed 's/\*/x/g' ) | |
| TEST_FOLDER_NAME=$( echo $TEST_FOLDER_NAME | sed 's/\:/_/g' ) | |
| echo "TEST_FOLDER_NAME=$TEST_FOLDER_NAME" >> "$GITHUB_ENV" | |
| - name: Upload Results Folder | |
| if: '!cancelled()' # always run this step, unless job manually cancelled | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| retention-days: 7 | |
| name: "test_results_${{ env.TEST_FOLDER_NAME }}" | |
| path: pmacct/test-framework/results | |
| # Step 5.5: Expose Results as Artifacts | |
| pytest-results: | |
| if: '!cancelled()' # always run this job, unless job manually cancelled | |
| needs: pytest-runtests | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Download Results Folder | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: test_results_* | |
| path: results | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Install Requirements | |
| run: | | |
| pip install --upgrade pip | |
| pip install pytest-html-merger | |
| - name: Merge results (test logs, global logs, and reports) in a single folder | |
| run: | | |
| mkdir test_results_merged | |
| cat results/**/pytestlog.log > test_results_merged/pytestlog.log | |
| cat results/**/monitor.log > test_results_merged/monitor.log | |
| find results/**/* -maxdepth 0 -type d -exec cp -r {} test_results_merged/ \; | |
| mkdir tmp_html | |
| randomname() { head -c16 /dev/urandom | base64 | tr -dc a-zA-Z; } | |
| for f in results/**/report.html; do mv "$f" tmp_html/`randomname`.html; done | |
| cp -R test_results_merged/assets tmp_html | |
| pytest_html_merger -i tmp_html -o test_results_merged/report_merged.html | |
| cp -R tmp_html/assets test_results_merged | |
| - name: Upload Merged Results Folder | |
| id: upload-artifact-results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| retention-days: 15 | |
| name: test_results | |
| path: test_results_merged | |
| # Workaround until they support multiple URLs deployments to github pages | |
| - name: Upload HTML report as standard artifact | |
| id: upload-artifact-html-report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| retention-days: 15 | |
| name: pytest_html_report | |
| path: test_results_merged/report_merged.html | |
| - name: Create folder to deploy to pages and fix permissions | |
| if: ${{ !cancelled() && github.ref == 'refs/heads/master' }} | |
| run: | | |
| mkdir github-pages | |
| cp test_results_merged/report_merged.html github-pages/index.html | |
| echo | |
| echo "Fix permissions (if necessary)..." | |
| chmod -c -R +rX github-pages | while read line; do | |
| echo "::warning title=Invalid file permissions automatically fixed::$line" | |
| done | |
| - name: Upload HTML report as github pages artifact (will be deployed by next job) | |
| if: ${{ !cancelled() && github.ref == 'refs/heads/master' }} | |
| uses: actions/upload-pages-artifact@v3 | |
| with: | |
| path: github-pages/ | |
| - name: Overwrite environment variables for the workflow_dispatch event | |
| run: | | |
| if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then | |
| echo "MARKER=${{ inputs.marker }}" >> $GITHUB_ENV | |
| if [ ! -z "${{ inputs.commit_id }}" ]; then | |
| echo "COMMIT_ID=${{ inputs.commit_id }}" >> $GITHUB_ENV | |
| fi | |
| echo "DEFAULT_ONLY=${{ inputs.default_only }}" >> $GITHUB_ENV | |
| fi | |
| - name: Add info to markdown summary | |
| env: | |
| MARKER: ${{ env.MARKER }} | |
| COMMIT_ID: ${{ env.COMMIT_ID }} | |
| DEFAULT_ONLY: ${{ env.DEFAULT_ONLY }} | |
| run: | | |
| echo "## :loudspeaker: Pytest Run Information: :loudspeaker:" >> $GITHUB_STEP_SUMMARY | |
| echo "### Test Results:" >> $GITHUB_STEP_SUMMARY | |
| echo "The Pytest HTML report is only deployed on github pages for runs triggered from the master branch (for security reasons), \ | |
| and is only available for the latest CI run. This is due to current Github Actions limitations of not supporting \ | |
| different URLs for deployments. Nonetheless, reports are anyway available for download as artifacts for up to 15 days \ | |
| after the test run (see Artifacts section below)." >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**Quick links for downloading:**" >> $GITHUB_STEP_SUMMARY | |
| echo "- Pytest HTML Report: ${{ steps.upload-artifact-html-report.outputs.artifact-url }}'" >> $GITHUB_STEP_SUMMARY | |
| echo "- Complete Pytest Results (with fine-grained logs for all tests): \ | |
| ${{ steps.upload-artifact-results.outputs.artifact-url }}" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo | |
| echo "### Input Arguments (can be changed when launching manual workflow_dispatch runs):" >> $GITHUB_STEP_SUMMARY | |
| echo "Test default scenarios only: $DEFAULT_ONLY" >> $GITHUB_STEP_SUMMARY | |
| echo "Pytest Marker: $MARKER" >> $GITHUB_STEP_SUMMARY | |
| echo "Commit ID: https://github.com/${{github.repository}}/tree/$COMMIT_ID" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo | |
| # Step 5.6: Deploy HTML report with github pages | |
| pytest-html-report-deploy: | |
| if: ${{ !cancelled() && github.ref == 'refs/heads/master' }} # We can only deploy pages from master (security reasons) | |
| needs: pytest-results | |
| runs-on: ubuntu-22.04 | |
| permissions: # Grant GITHUB_TOKEN the permissions required to make a Pages deployment | |
| pages: write # --> to deploy to Pages | |
| id-token: write # --> to verify the deployment originates from an appropriate source | |
| environment: | |
| name: github-pages | |
| url: ${{ steps.deployment.outputs.page_url }} # The deployment URL cannot be changed (for now...) | |
| steps: | |
| - name: Deploy artifact to Github Pages | |
| uses: actions/deploy-pages@v4 | |
| id: deployment |