Skip to content

Commit 789be1c

Browse files
firewavefirewave
authored
Tokenizer: moved test-only tokenize() to SimpleTokenizer / several cleanups (danmar#6264)
1 parent f450285 commit 789be1c

45 files changed

Lines changed: 1229 additions & 1470 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Makefile‎

Lines changed: 45 additions & 45 deletions
Large diffs are not rendered by default.

‎lib/tokenize.cpp‎

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3439,16 +3439,6 @@ bool Tokenizer::simplifyTokens1(const std::string &configuration)
34393439
return true;
34403440
}
34413441

3442-
// cppcheck-suppress unusedFunction - used in tests only
3443-
bool Tokenizer::tokenize(std::istream &code,
3444-
const char FileName[],
3445-
const std::string &configuration)
3446-
{
3447-
if (!list.createTokens(code, FileName))
3448-
return false;
3449-
3450-
return simplifyTokens1(configuration);
3451-
}
34523442
//---------------------------------------------------------------------------
34533443

34543444
void Tokenizer::findComplicatedSyntaxErrorsInTemplates()
@@ -10659,6 +10649,7 @@ void Tokenizer::simplifyNamespaceAliases()
1065910649
}
1066010650
}
1066110651

10652+
// TODO: how to move the Preprocessor dependency out of here?
1066210653
bool Tokenizer::hasIfdef(const Token *start, const Token *end) const
1066310654
{
1066410655
assert(mPreprocessor);

‎lib/tokenize.h‎

Lines changed: 0 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -79,30 +79,6 @@ class CPPCHECKLIB Tokenizer {
7979
bool isScopeNoReturn(const Token *endScopeToken, bool *unknown = nullptr) const;
8080

8181
bool simplifyTokens1(const std::string &configuration);
82-
/**
83-
* Tokenize code
84-
* @param code input stream for code, e.g.
85-
* \code
86-
* #file "p.h"
87-
* class Foo
88-
* {
89-
* private:
90-
* void Bar();
91-
* };
92-
*
93-
* #endfile
94-
* void Foo::Bar()
95-
* {
96-
* }
97-
* \endcode
98-
*
99-
* @param FileName The filename
100-
* @param configuration E.g. "A" for code where "#ifdef A" is true
101-
* @return false if source code contains syntax errors
102-
*/
103-
bool tokenize(std::istream &code,
104-
const char FileName[],
105-
const std::string &configuration = emptyString);
10682

10783
private:
10884
/** Set variable id */

‎test/helpers.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@
4444

4545
class SuppressionList;
4646

47+
const Settings SimpleTokenizer::s_settings;
48+
const Preprocessor SimpleTokenizer::s_preprocessor{s_settings, nullptr}; // TODO: provide ErrorLogger
49+
4750
// TODO: better path-only usage
4851
ScopedFile::ScopedFile(std::string name, const std::string &content, std::string path)
4952
: mName(std::move(name))

‎test/helpers.h‎

Lines changed: 42 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
#ifndef helpersH
2020
#define helpersH
2121

22+
#include "preprocessor.h"
2223
#include "settings.h"
2324
#include "standards.h"
2425
#include "tokenize.h"
@@ -31,34 +32,62 @@
3132
#include <vector>
3233

3334
class Token;
34-
class Preprocessor;
3535
class SuppressionList;
3636
class ErrorLogger;
3737
namespace simplecpp {
3838
struct DUI;
3939
}
4040

41-
class SimpleTokenizer {
41+
// TODO: make Tokenizer private
42+
class SimpleTokenizer : public Tokenizer {
4243
public:
43-
SimpleTokenizer(ErrorLogger& errorlogger, const char sample[], bool cpp = true)
44-
: tokenizer{settings, &errorlogger}
44+
SimpleTokenizer(ErrorLogger& errorlogger, const char code[], bool cpp = true)
45+
: Tokenizer{s_settings, &errorlogger, &s_preprocessor}
4546
{
46-
std::istringstream iss(sample);
47-
if (!tokenizer.tokenize(iss, cpp ? "test.cpp" : "test.c"))
47+
if (!tokenize(code, cpp))
4848
throw std::runtime_error("creating tokens failed");
4949
}
5050

51-
Token* tokens() {
52-
return tokenizer.tokens();
53-
}
51+
SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger)
52+
: Tokenizer{settings, &errorlogger, &s_preprocessor}
53+
{}
54+
55+
SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger, const Preprocessor* preprocessor)
56+
: Tokenizer{settings, &errorlogger, preprocessor}
57+
{}
58+
59+
/*
60+
Token* tokens() {
61+
return Tokenizer::tokens();
62+
}
63+
64+
const Token* tokens() const {
65+
return Tokenizer::tokens();
66+
}
67+
*/
68+
69+
/**
70+
* Tokenize code
71+
* @param code The code
72+
* @param cpp Indicates if the code is C++
73+
* @param configuration E.g. "A" for code where "#ifdef A" is true
74+
* @return false if source code contains syntax errors
75+
*/
76+
bool tokenize(const char code[],
77+
bool cpp = true,
78+
const std::string &configuration = emptyString)
79+
{
80+
std::istringstream istr(code);
81+
if (!list.createTokens(istr, cpp ? "test.cpp" : "test.c"))
82+
return false;
5483

55-
const Token* tokens() const {
56-
return tokenizer.tokens();
84+
return simplifyTokens1(configuration);
5785
}
5886

5987
private:
60-
const Settings settings;
61-
Tokenizer tokenizer;
88+
// TODO. find a better solution
89+
static const Settings s_settings;
90+
static const Preprocessor s_preprocessor;
6291
};
6392

6493
class SimpleTokenList

‎test/test64bit.cpp‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,11 @@
1616
* along with this program. If not, see <http://www.gnu.org/licenses/>.
1717
*/
1818

19-
2019
#include "check64bit.h"
2120
#include "errortypes.h"
22-
#include "settings.h"
2321
#include "fixture.h"
24-
#include "tokenize.h"
25-
26-
#include <sstream>
22+
#include "helpers.h"
23+
#include "settings.h"
2724

2825
class Test64BitPortability : public TestFixture {
2926
public:
@@ -45,9 +42,8 @@ class Test64BitPortability : public TestFixture {
4542
#define check(code) check_(code, __FILE__, __LINE__)
4643
void check_(const char code[], const char* file, int line) {
4744
// Tokenize..
48-
Tokenizer tokenizer(settings, this);
49-
std::istringstream istr(code);
50-
ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
45+
SimpleTokenizer tokenizer(settings, *this);
46+
ASSERT_LOC(tokenizer.tokenize(code), file, line);
5147

5248
// Check char variable usage..
5349
Check64BitPortability check64BitPortability(&tokenizer, &settings, this);

‎test/testassert.cpp‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,9 @@
1919

2020
#include "checkassert.h"
2121
#include "errortypes.h"
22-
#include "settings.h"
2322
#include "fixture.h"
24-
#include "tokenize.h"
25-
26-
#include <sstream>
27-
23+
#include "helpers.h"
24+
#include "settings.h"
2825

2926
class TestAssert : public TestFixture {
3027
public:
@@ -34,11 +31,10 @@ class TestAssert : public TestFixture {
3431
const Settings settings = settingsBuilder().severity(Severity::warning).build();
3532

3633
#define check(...) check_(__FILE__, __LINE__, __VA_ARGS__)
37-
void check_(const char* file, int line, const char code[], const char *filename = "test.cpp") {
34+
void check_(const char* file, int line, const char code[]) {
3835
// Tokenize..
39-
Tokenizer tokenizer(settings, this);
40-
std::istringstream istr(code);
41-
ASSERT_LOC(tokenizer.tokenize(istr, filename), file, line);
36+
SimpleTokenizer tokenizer(settings, *this);
37+
ASSERT_LOC(tokenizer.tokenize(code), file, line);
4238

4339
// Check..
4440
runChecks<CheckAssert>(tokenizer, this);

‎test/testastutils.cpp‎

Lines changed: 22 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -16,18 +16,16 @@
1616
* along with this program. If not, see <http://www.gnu.org/licenses/>.
1717
*/
1818

19-
2019
#include "astutils.h"
20+
#include "fixture.h"
21+
#include "helpers.h"
2122
#include "library.h"
2223
#include "settings.h"
23-
#include "fixture.h"
2424
#include "symboldatabase.h"
2525
#include "token.h"
26-
#include "tokenize.h"
2726
#include "tokenlist.h"
2827

2928
#include <cstring>
30-
#include <sstream>
3129

3230
class TestAstUtils : public TestFixture {
3331
public:
@@ -54,9 +52,8 @@ class TestAstUtils : public TestFixture {
5452
#define findLambdaEndToken(...) findLambdaEndToken_(__FILE__, __LINE__, __VA_ARGS__)
5553
bool findLambdaEndToken_(const char* file, int line, const char code[], const char pattern[] = nullptr, bool checkNext = true) {
5654
const Settings settings;
57-
Tokenizer tokenizer(settings, this);
58-
std::istringstream istr(code);
59-
ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
55+
SimpleTokenizer tokenizer(settings, *this);
56+
ASSERT_LOC(tokenizer.tokenize(code), file, line);
6057
const Token* const tokStart = pattern ? Token::findsimplematch(tokenizer.tokens(), pattern, strlen(pattern)) : tokenizer.tokens();
6158
const Token * const tokEnd = (::findLambdaEndToken)(tokStart);
6259
return tokEnd && (!checkNext || tokEnd->next() == nullptr);
@@ -92,9 +89,8 @@ class TestAstUtils : public TestFixture {
9289

9390
#define findLambdaStartToken(code) findLambdaStartToken_(code, __FILE__, __LINE__)
9491
bool findLambdaStartToken_(const char code[], const char* file, int line) {
95-
Tokenizer tokenizer(settingsDefault, this);
96-
std::istringstream istr(code);
97-
ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
92+
SimpleTokenizer tokenizer(settingsDefault, *this);
93+
ASSERT_LOC(tokenizer.tokenize(code), file, line);
9894
const Token * const tokStart = (::findLambdaStartToken)(tokenizer.list.back());
9995
return tokStart && tokStart == tokenizer.list.front();
10096
}
@@ -124,9 +120,8 @@ class TestAstUtils : public TestFixture {
124120

125121
#define isNullOperand(code) isNullOperand_(code, __FILE__, __LINE__)
126122
bool isNullOperand_(const char code[], const char* file, int line) {
127-
Tokenizer tokenizer(settingsDefault, this);
128-
std::istringstream istr(code);
129-
ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
123+
SimpleTokenizer tokenizer(settingsDefault, *this);
124+
ASSERT_LOC(tokenizer.tokenize(code), file, line);
130125
return (::isNullOperand)(tokenizer.tokens());
131126
}
132127

@@ -145,9 +140,8 @@ class TestAstUtils : public TestFixture {
145140

146141
#define isReturnScope(code, offset) isReturnScope_(code, offset, __FILE__, __LINE__)
147142
bool isReturnScope_(const char code[], int offset, const char* file, int line) {
148-
Tokenizer tokenizer(settingsDefault, this);
149-
std::istringstream istr(code);
150-
ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
143+
SimpleTokenizer tokenizer(settingsDefault, *this);
144+
ASSERT_LOC(tokenizer.tokenize(code), file, line);
151145
const Token * const tok = (offset < 0)
152146
? tokenizer.list.back()->tokAt(1+offset)
153147
: tokenizer.tokens()->tokAt(offset);
@@ -176,9 +170,8 @@ class TestAstUtils : public TestFixture {
176170
#define isSameExpression(...) isSameExpression_(__FILE__, __LINE__, __VA_ARGS__)
177171
bool isSameExpression_(const char* file, int line, const char code[], const char tokStr1[], const char tokStr2[], bool cpp) {
178172
Library library;
179-
Tokenizer tokenizer(settingsDefault, this);
180-
std::istringstream istr(code);
181-
ASSERT_LOC(tokenizer.tokenize(istr, cpp ? "test.cpp" : "test.c"), file, line);
173+
SimpleTokenizer tokenizer(settingsDefault, *this);
174+
ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line);
182175
const Token * const tok1 = Token::findsimplematch(tokenizer.tokens(), tokStr1, strlen(tokStr1));
183176
const Token * const tok2 = Token::findsimplematch(tok1->next(), tokStr2, strlen(tokStr2));
184177
return (isSameExpression)(false, tok1, tok2, library, false, true, nullptr);
@@ -224,9 +217,8 @@ class TestAstUtils : public TestFixture {
224217

225218
#define isVariableChanged(code, startPattern, endPattern) isVariableChanged_(code, startPattern, endPattern, __FILE__, __LINE__)
226219
bool isVariableChanged_(const char code[], const char startPattern[], const char endPattern[], const char* file, int line) {
227-
Tokenizer tokenizer(settingsDefault, this);
228-
std::istringstream istr(code);
229-
ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
220+
SimpleTokenizer tokenizer(settingsDefault, *this);
221+
ASSERT_LOC(tokenizer.tokenize(code), file, line);
230222
const Token * const tok1 = Token::findsimplematch(tokenizer.tokens(), startPattern, strlen(startPattern));
231223
const Token * const tok2 = Token::findsimplematch(tokenizer.tokens(), endPattern, strlen(endPattern));
232224
return (isVariableChanged)(tok1, tok2, 1, false, &settingsDefault);
@@ -258,9 +250,8 @@ class TestAstUtils : public TestFixture {
258250

259251
#define isVariableChangedByFunctionCall(code, pattern, inconclusive) isVariableChangedByFunctionCall_(code, pattern, inconclusive, __FILE__, __LINE__)
260252
bool isVariableChangedByFunctionCall_(const char code[], const char pattern[], bool *inconclusive, const char* file, int line) {
261-
Tokenizer tokenizer(settingsDefault, this);
262-
std::istringstream istr(code);
263-
ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
253+
SimpleTokenizer tokenizer(settingsDefault, *this);
254+
ASSERT_LOC(tokenizer.tokenize(code), file, line);
264255
const Token * const argtok = Token::findmatch(tokenizer.tokens(), pattern);
265256
ASSERT_LOC(argtok, file, line);
266257
int indirect = (argtok->variable() && argtok->variable()->isArray());
@@ -401,9 +392,8 @@ class TestAstUtils : public TestFixture {
401392
int line)
402393
{
403394
const Settings settings = settingsBuilder().library("std.cfg").build();
404-
Tokenizer tokenizer(settings, this);
405-
std::istringstream istr(code);
406-
ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
395+
SimpleTokenizer tokenizer(settings, *this);
396+
ASSERT_LOC(tokenizer.tokenize(code), file, line);
407397
const Token* const start = Token::findsimplematch(tokenizer.tokens(), startPattern, strlen(startPattern));
408398
const Token* const end = Token::findsimplematch(start, endPattern, strlen(endPattern));
409399
const Token* const expr = Token::findsimplematch(tokenizer.tokens(), var, strlen(var));
@@ -432,9 +422,8 @@ class TestAstUtils : public TestFixture {
432422

433423
#define nextAfterAstRightmostLeaf(code, parentPattern, rightPattern) nextAfterAstRightmostLeaf_(code, parentPattern, rightPattern, __FILE__, __LINE__)
434424
bool nextAfterAstRightmostLeaf_(const char code[], const char parentPattern[], const char rightPattern[], const char* file, int line) {
435-
Tokenizer tokenizer(settingsDefault, this);
436-
std::istringstream istr(code);
437-
ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
425+
SimpleTokenizer tokenizer(settingsDefault, *this);
426+
ASSERT_LOC(tokenizer.tokenize(code), file, line);
438427
const Token * tok = Token::findsimplematch(tokenizer.tokens(), parentPattern, strlen(parentPattern));
439428
return Token::simpleMatch((::nextAfterAstRightmostLeaf)(tok), rightPattern, strlen(rightPattern));
440429
}
@@ -456,9 +445,8 @@ class TestAstUtils : public TestFixture {
456445
enum class Result {False, True, Fail};
457446

458447
Result isUsedAsBool(const char code[], const char pattern[]) {
459-
Tokenizer tokenizer(settingsDefault, this);
460-
std::istringstream istr(code);
461-
if (!tokenizer.tokenize(istr, "test.cpp"))
448+
SimpleTokenizer tokenizer(settingsDefault, *this);
449+
if (!tokenizer.tokenize(code))
462450
return Result::Fail;
463451
const Token * const argtok = Token::findmatch(tokenizer.tokens(), pattern);
464452
if (!argtok)

‎test/testautovariables.cpp‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,11 @@
1616
* along with this program. If not, see <http://www.gnu.org/licenses/>.
1717
*/
1818

19-
2019
#include "checkautovariables.h"
2120
#include "errortypes.h"
22-
#include "settings.h"
2321
#include "fixture.h"
24-
#include "tokenize.h"
25-
26-
#include <sstream>
22+
#include "helpers.h"
23+
#include "settings.h"
2724

2825
class TestAutoVariables : public TestFixture {
2926
public:
@@ -33,13 +30,12 @@ class TestAutoVariables : public TestFixture {
3330
const Settings settings = settingsBuilder().severity(Severity::warning).severity(Severity::style).library("std.cfg").library("qt.cfg").build();
3431

3532
#define check(...) check_(__FILE__, __LINE__, __VA_ARGS__)
36-
void check_(const char* file, int line, const char code[], bool inconclusive = true, const char* filename = "test.cpp") {
33+
void check_(const char* file, int line, const char code[], bool inconclusive = true, bool cpp = true) {
3734
const Settings settings1 = settingsBuilder(settings).certainty(Certainty::inconclusive, inconclusive).build();
3835

3936
// Tokenize..
40-
Tokenizer tokenizer(settings1, this);
41-
std::istringstream istr(code);
42-
ASSERT_LOC(tokenizer.tokenize(istr, filename), file, line);
37+
SimpleTokenizer tokenizer(settings1, *this);
38+
ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line);
4339

4440
runChecks<CheckAutoVariables>(tokenizer, this);
4541
}
@@ -916,7 +912,7 @@ class TestAutoVariables : public TestFixture {
916912
check("void svn_repos_dir_delta2() {\n"
917913
" struct context c;\n"
918914
" SVN_ERR(delete(&c, root_baton, src_entry, pool));\n"
919-
"}\n", false, "test.c");
915+
"}\n", false, /* cpp= */ false);
920916
ASSERT_EQUALS("", errout_str());
921917
}
922918

0 commit comments

Comments
 (0)