Skip to content

Commit 052de79

Browse files
firewavefirewave
authored
some test/CI related refactoring and cleanup (cppcheck-opensource#3163)
1 parent c45b941 commit 052de79

12 files changed

Lines changed: 172 additions & 142 deletions

‎.github/workflows/CI-unixish-docker.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,10 @@ jobs:
6565
run: |
6666
make -j$(nproc) check HAVE_RULES=yes
6767
68+
- name: Run extra tests
69+
run: |
70+
tools/generate_and_run_more_tests.sh
71+
6872
- name: Validate
6973
run: |
7074
make -j$(nproc) checkCWEEntries validateXML

‎.github/workflows/CI-unixish.yml‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,28 +17,36 @@ jobs:
1717
steps:
1818
- uses: actions/checkout@v2
1919

20+
- name: Set up Python 3.9
21+
uses: actions/setup-python@v2
22+
with:
23+
python-version: 3.9
24+
2025
- name: Install missing software on ubuntu 18.04
2126
if: matrix.os == 'ubuntu-18.04'
2227
run: |
2328
sudo apt-get update
2429
sudo apt-get install libxml2-utils
2530
sudo apt-get install z3 libz3-dev
2631
cp externals/z3_version_old.h externals/z3_version.h
27-
sudo apt-get install python3-pytest
2832
2933
- name: Install missing software on ubuntu 20.04
3034
if: matrix.os == 'ubuntu-20.04'
3135
run: |
3236
sudo apt-get update
3337
sudo apt-get install libxml2-utils
3438
sudo apt-get install z3 libz3-dev
35-
sudo apt-get install python3-pytest
3639
3740
- name: Install missing software on macos
3841
if: contains(matrix.os, 'macos')
3942
run: |
4043
brew install coreutils z3
4144
cp externals/z3_version_old.h externals/z3_version.h
45+
46+
- name: Install missing Python packages
47+
run: |
48+
python -m pip install pip --upgrade
49+
python -m pip install pytest
4250
4351
- name: Install Qt
4452
if: contains(matrix.os, 'ubuntu')
@@ -77,13 +85,20 @@ jobs:
7785
run: |
7886
make -j$(nproc) check USE_Z3=yes HAVE_RULES=yes
7987
80-
- name: Run test/cli
88+
# the script uses sed parameters not supported by MacOS
89+
- name: Run extra tests
8190
if: contains(matrix.os, 'ubuntu')
8291
run: |
83-
cd ..
84-
ln -s cppcheck cpp\ check
85-
cd cpp\ check/test/cli
86-
pytest-3 test-*.py
92+
tools/generate_and_run_more_tests.sh
93+
94+
- name: Run test/cli
95+
run: |
96+
cd test/cli
97+
pytest test-*.py
98+
cd ../../..
99+
ln -s cppcheck 'cpp check'
100+
cd 'cpp check/test/cli'
101+
pytest test-*.py
87102
88103
- name: Validate
89104
run: |

‎.github/workflows/CI-windows.yml‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,18 @@ jobs:
1818
# windows 2016 should default to VS 2017. Not supported by setup-msbuild
1919
os: [windows-2019]
2020
arch: [x64, x86]
21-
fail-fast: true
21+
fail-fast: false
2222

2323
runs-on: ${{ matrix.os }}
2424

2525
steps:
2626
- uses: actions/checkout@v2
2727

28+
- name: Set up Python 3.9
29+
uses: actions/setup-python@v2
30+
with:
31+
python-version: 3.9
32+
2833
- name: Setup msbuild.exe
2934
uses: microsoft/[email protected]
3035

@@ -62,6 +67,12 @@ jobs:
6267
with:
6368
modules: 'qtcharts'
6469

70+
- name: Install missing Python packages
71+
run: |
72+
python -m pip install pip --upgrade
73+
python -m pip install pytest
74+
python -m pip install pytest-custom_exit_code
75+
6576
- name: Build GUI release
6677
if: matrix.arch == 'x64'
6778
run: |
@@ -100,4 +111,16 @@ jobs:
100111
101112
- name: Run Release test
102113
run: .\bin\testrunner.exe || exit /b !errorlevel!
103-
114+
115+
- name: Run test/cli
116+
run: |
117+
:: since FILESDIR is not set copy the binary to the root so the addons are found
118+
copy .\bin\cppcheck.exe .\cppcheck.exe || exit /b !errorlevel!
119+
copy .\bin\cppcheck-core.dll .\cppcheck-core.dll || exit /b !errorlevel!
120+
cd test/cli || exit /b !errorlevel!
121+
python -m pytest --suppress-no-test-exit-code test-clang-import.py || exit /b !errorlevel!
122+
python -m pytest test-helloworld.py || exit /b !errorlevel!
123+
python -m pytest test-inline-suppress.py || exit /b !errorlevel!
124+
python -m pytest test-more-projects.py || exit /b !errorlevel!
125+
python -m pytest test-proj2.py || exit /b !errorlevel!
126+
python -m pytest test-suppress-syntaxError.py || exit /b !errorlevel!

‎.github/workflows/scriptcheck.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ jobs:
3030
- name: Install missing software on ubuntu (Python 3)
3131
if: matrix.python-version != '2.7'
3232
run: |
33+
sudo apt-get update
3334
sudo apt-get install shellcheck libxml2-utils
3435
python -m pip install pip --upgrade
3536
python -m pip install natsort
@@ -76,6 +77,10 @@ jobs:
7677
run: |
7778
python -m compileall ./addons
7879
80+
- name: test matchcompiler
81+
run: |
82+
python tools/test_matchcompiler.py
83+
7984
- name: test addons
8085
run: |
8186
make -j$(nproc) -s

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
*.pyc
55
cppcheck
66
cppcheck.exe
7+
cppcheck-core.dll
78
dmake
89
dmake.exe
910
reduce

‎.travis.yml‎

Lines changed: 1 addition & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -45,14 +45,8 @@ matrix:
4545
compiler: clang
4646
script:
4747
- make -j$(nproc) -s
48-
# run extra tests
49-
- tools/generate_and_run_more_tests.sh
50-
# check for missing CWE entries
51-
- make checkCWEEntries
52-
# check cfg files
48+
# check cfg files
5349
- make checkcfg
54-
# Validate Result XML
55-
- make validateXML
5650
# check htmlreport stuff
5751
- python2 htmlreport/test_htmlreport.py
5852
- python3 htmlreport/test_htmlreport.py
@@ -74,9 +68,6 @@ matrix:
7468
- /usr/bin/cppcheck ./cli
7569
# check if showtime=top5 works
7670
- ./tools/test_showtimetop5.sh
77-
# check matchcompiler
78-
- python2 tools/test_matchcompiler.py
79-
- python3 tools/test_matchcompiler.py
8071
# check --dump
8172
- ${CPPCHECK} test/testpreprocessor.cpp --dump
8273
- xmllint --noout test/testpreprocessor.cpp.dump
@@ -151,6 +142,3 @@ script:
151142
# compile cppcheck, default build
152143
- echo $CXXFLAGS
153144
- make -s check -j$(nproc)
154-
# Testing addons (disabled 2020-11-24 because Travis fails, TODO try to enable these)
155-
# - PYTHONPATH=./addons python -m pytest addons/test/test-*.py
156-
# - PYTHONPATH=./addons python3 -m pytest addons/test/test-*.py

‎appveyor.yml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -67,9 +67,4 @@ test_script:
6767
- build\bin\testrunner.exe -q
6868
- IF EXIST bin\debug\testrunner.exe bin\debug\testrunner.exe -q
6969
- IF EXIST bin\testrunner.exe bin\testrunner.exe -q
70-
- cd test\cli
71-
- python -m pytest test-helloworld.py
72-
- python -m pytest test-inline-suppress.py
73-
- python -m pytest test-proj2.py
74-
- python -m pytest test-suppress-syntaxError.py
7570
- 'IF defined cygwin_build C:\cygwin64\bin\bash -e -l -c "cd /cygdrive/c/projects/cppcheck && make clean && make -j 2 test checkcfg"'

‎test/cli/test-clang-import.py‎

Lines changed: 19 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,14 @@
44
import os
55
import re
66
import subprocess
7+
import pytest
78
from testutils import cppcheck
89

10+
try:
11+
subprocess.call(['clang', '--version'])
12+
except OSError:
13+
pytest.skip("'clang' does not exist", allow_module_level=True)
14+
915

1016
def get_debug_section(title, stdout):
1117
s = re.sub(r'0x[0-9a-fA-F]+', '0x12345678', stdout)
@@ -21,7 +27,7 @@ def get_debug_section(title, stdout):
2127
s = re.sub(r"return '[a-zA-Z0-9: *]+'", "return", s)
2228

2329
pos1 = s.find(title)
24-
assert pos1 > 0
30+
assert pos1 > 0, 'title not found'
2531
pos1 = s.find('\n', pos1) + 1
2632
assert pos1 > 0
2733
pos2 = s.find("\n##", pos1)
@@ -31,50 +37,38 @@ def get_debug_section(title, stdout):
3137

3238

3339
def check_symbol_database(code):
34-
# Only compare symboldatabases if clang is found in PATH
35-
try:
36-
subprocess.call(['clang', '--version'])
37-
except OSError:
38-
return
39-
4040
testfile = 'test.cpp'
4141
with open(testfile, 'w+t') as f:
4242
f.write(code)
43-
ret1, stdout1, stderr1 = cppcheck(['--clang', '--debug', '-v', testfile])
44-
ret2, stdout2, stderr2 = cppcheck(['--debug', '-v', testfile])
43+
ret1, stdout1, _ = cppcheck(['--clang', '--debug', '-v', testfile])
44+
ret2, stdout2, _ = cppcheck(['--debug', '-v', testfile])
4545
os.remove(testfile)
46+
assert 0 == ret1, stdout1
47+
assert 0 == ret2, stdout2
4648
assert get_debug_section('### Symbol database', stdout1) == get_debug_section('### Symbol database', stdout2)
4749

4850

4951
def check_ast(code):
50-
# Only compare syntax trees if clang is found in PATH
51-
try:
52-
subprocess.call(['clang', '--version'])
53-
except OSError:
54-
return
55-
5652
testfile = 'test.cpp'
5753
with open(testfile, 'w+t') as f:
5854
f.write(code)
59-
ret1, stdout1, stderr1 = cppcheck(['--clang', '--debug', '-v', testfile])
60-
ret2, stdout2, stderr2 = cppcheck(['--debug', '-v', testfile])
55+
ret1, stdout1, _ = cppcheck(['--clang', '--debug', '-v', testfile])
56+
ret2, stdout2, _ = cppcheck(['--debug', '-v', testfile])
6157
os.remove(testfile)
58+
assert 0 == ret1, stdout1
59+
assert 0 == ret2, stdout1
6260
assert get_debug_section('##AST', stdout1) == get_debug_section('##AST', stdout2)
6361

6462

6563
def todo_check_ast(code):
66-
# Only compare syntax trees if clang is found in PATH
67-
try:
68-
subprocess.call(['clang', '--version'])
69-
except OSError:
70-
return
71-
7264
testfile = 'test.cpp'
7365
with open(testfile, 'w+t') as f:
7466
f.write(code)
75-
ret1, stdout1, stderr1 = cppcheck(['--clang', '--debug', '-v', testfile])
76-
ret2, stdout2, stderr2 = cppcheck(['--debug', '-v', testfile])
67+
ret1, stdout1, _ = cppcheck(['--clang', '--debug', '-v', testfile])
68+
ret2, stdout2, _ = cppcheck(['--debug', '-v', testfile])
7769
os.remove(testfile)
70+
assert 0 == ret1, stdout1
71+
assert 0 == ret2, stdout2
7872
assert get_debug_section('##AST', stdout1) != get_debug_section('##AST', stdout2)
7973

8074

0 commit comments

Comments
 (0)