Skip to content

Commit a5664c3

Browse files
shaneasddanmar
shaneasd
authored and
danmar
committed
allow token iteration in range for
1 parent 08626b9 commit a5664c3

8 files changed

Lines changed: 222 additions & 0 deletions

File tree

‎lib/cppcheck.vcxproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,7 @@
160160
<ClInclude Include="token.h" />
161161
<ClInclude Include="tokenize.h" />
162162
<ClInclude Include="tokenlist.h" />
163+
<ClInclude Include="tokenrange.h" />
163164
<ClInclude Include="utils.h" />
164165
<ClInclude Include="valueflow.h" />
165166
<ClInclude Include="version.h" />

‎lib/cppcheck.vcxproj.filters‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -361,6 +361,9 @@
361361
<ClInclude Include="bughuntingchecks.h">
362362
<Filter>Header Files</Filter>
363363
</ClInclude>
364+
<ClInclude Include="tokenrange.h">
365+
<Filter>Header Files</Filter>
366+
</ClInclude>
364367
<ClInclude Include="errortypes.h">
365368
<Filter>Header Files</Filter>
366369
</ClInclude>

‎lib/token.cpp‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include "symboldatabase.h"
2525
#include "tokenlist.h"
2626
#include "utils.h"
27+
#include "tokenrange.h"
2728
#include "valueflow.h"
2829

2930
#include <algorithm>
@@ -55,6 +56,16 @@ Token::~Token()
5556
delete mImpl;
5657
}
5758

59+
/*
60+
* Get a TokenRange which starts at this token and contains every token following it in order up to but not including 't'
61+
* e.g. for the sequence of tokens A B C D E, C.until(E) would yield the Range C D
62+
* note t can be nullptr to iterate all the way to the end.
63+
*/
64+
ConstTokenRange Token::until(const Token* t) const
65+
{
66+
return ConstTokenRange(this, t);
67+
}
68+
5869
static const std::unordered_set<std::string> controlFlowKeywords = {
5970
"goto",
6071
"do",

‎lib/token.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ class ValueType;
4444
class Variable;
4545
class TokenList;
4646

47+
class ConstTokenRange;
48+
4749
/**
4850
* @brief This struct stores pointers to the front and back tokens of the list this token is in.
4951
*/
@@ -187,6 +189,8 @@ class CPPCHECKLIB Token {
187189
explicit Token(TokensFrontBack *tokensFrontBack = nullptr);
188190
~Token();
189191

192+
ConstTokenRange until(const Token * t) const;
193+
190194
template<typename T>
191195
void str(T&& s) {
192196
mStr = s;

‎lib/tokenrange.h‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
/*
2+
* Cppcheck - A tool for static C/C++ code analysis
3+
* Copyright (C) 2007-2020 Cppcheck team.
4+
*
5+
* This program is free software: you can redistribute it and/or modify
6+
* it under the terms of the GNU General Public License as published by
7+
* the Free Software Foundation, either version 3 of the License, or
8+
* (at your option) any later version.
9+
*
10+
* This program is distributed in the hope that it will be useful,
11+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+
* GNU General Public License for more details.
14+
*
15+
* You should have received a copy of the GNU General Public License
16+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
17+
*/
18+
19+
//---------------------------------------------------------------------------
20+
#ifndef tokenrangeH
21+
#define tokenrangeH
22+
//---------------------------------------------------------------------------
23+
24+
#include "config.h"
25+
26+
template<typename T, REQUIRES("T must be a Token class", std::is_convertible<T*, const Token*>)>
27+
class TokenRangeBase
28+
{
29+
T* mFront;
30+
T* mBack;
31+
32+
public:
33+
TokenRangeBase(T* front, T* back) : mFront(front), mBack(back) {}
34+
35+
struct TokenIterator
36+
{
37+
using iterator_category = std::forward_iterator_tag;
38+
using value_type = T*;
39+
using difference_type = std::ptrdiff_t;
40+
using pointer = void;
41+
using reference = T*;
42+
43+
T* mt;
44+
TokenIterator() : mt(nullptr) {}
45+
explicit TokenIterator(T* t) : mt(t) {}
46+
TokenIterator& operator++() { mt = mt->next(); return *this; }
47+
bool operator==(const TokenIterator& b) const { return mt == b.mt; }
48+
bool operator!=(const TokenIterator& b) const { return mt != b.mt; }
49+
T* operator*() const { return mt; }
50+
};
51+
52+
TokenIterator begin() const { return TokenIterator(mFront); }
53+
TokenIterator end() const { return TokenIterator(mBack); }
54+
};
55+
56+
class TokenRange : public TokenRangeBase<Token> {
57+
public:
58+
TokenRange(Token* front, Token* back) : TokenRangeBase<Token>(front, back) {}
59+
};
60+
61+
class ConstTokenRange : public TokenRangeBase<const Token> {
62+
public:
63+
ConstTokenRange(const Token* front, const Token* back) : TokenRangeBase<const Token>(front, back) {}
64+
};
65+
66+
#endif // tokenrangeH

‎test/testrunner.vcxproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@
8888
<ClCompile Include="testtoken.cpp" />
8989
<ClCompile Include="testtokenize.cpp" />
9090
<ClCompile Include="testtokenlist.cpp" />
91+
<ClCompile Include="testtokenrange.cpp" />
9192
<ClCompile Include="testtype.cpp" />
9293
<ClCompile Include="testuninitvar.cpp" />
9394
<ClCompile Include="testunusedfunctions.cpp" />

‎test/testrunner.vcxproj.filters‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,9 @@
223223
<ClCompile Include="testsummaries.cpp">
224224
<Filter>Source Files</Filter>
225225
</ClCompile>
226+
<ClCompile Include="testtokenrange.cpp">
227+
<Filter>Source Files</Filter>
228+
</ClCompile>
226229
</ItemGroup>
227230
<ItemGroup>
228231
<ClInclude Include="options.h">

‎test/testtokenrange.cpp‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
/*
2+
* Cppcheck - A tool for static C/C++ code analysis
3+
* Copyright (C) 2007-2020 Cppcheck team.
4+
*
5+
* This program is free software: you can redistribute it and/or modify
6+
* it under the terms of the GNU General Public License as published by
7+
* the Free Software Foundation, either version 3 of the License, or
8+
* (at your option) any later version.
9+
*
10+
* This program is distributed in the hope that it will be useful,
11+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+
* GNU General Public License for more details.
14+
*
15+
* You should have received a copy of the GNU General Public License
16+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
17+
*/
18+
19+
#include "settings.h"
20+
#include "testsuite.h"
21+
#include "testutils.h"
22+
#include "token.h"
23+
#include "tokenize.h"
24+
#include "tokenlist.h"
25+
#include "tokenrange.h"
26+
#include "symboldatabase.h"
27+
28+
#include <string>
29+
#include <vector>
30+
31+
struct InternalError;
32+
33+
34+
class TestTokenRange : public TestFixture {
35+
public:
36+
TestTokenRange() : TestFixture("TestTokenRange") {
37+
}
38+
39+
private:
40+
void run() OVERRIDE {
41+
TEST_CASE(enumerationToEnd);
42+
TEST_CASE(untilHelperToEnd);
43+
TEST_CASE(untilHelperPartWay);
44+
TEST_CASE(partialEnumeration);
45+
TEST_CASE(scopeExample);
46+
TEST_CASE(exampleAlgorithms);
47+
}
48+
49+
std::string testTokenRange(ConstTokenRange range, const Token* start, const Token* end) const
50+
{
51+
auto tokenToString = [](const Token* t) { return t ? t->str() : "<null>"; };
52+
int index = 0;
53+
const Token* expected = start;
54+
for (const Token* t : range)
55+
{
56+
if (expected != t) {
57+
std::ostringstream message;
58+
message << "Failed to match token " << tokenToString(expected) << " at position " << index << ". Got " << tokenToString(t) << " instead";
59+
return message.str();
60+
}
61+
index++;
62+
expected = expected->next();
63+
}
64+
if (expected != end) {
65+
std::ostringstream message;
66+
message << "Failed to terminate on " << tokenToString(end) << ". Instead terminated on " << tokenToString(expected) << " at position " << index << ".";
67+
return message.str();
68+
}
69+
return "";
70+
}
71+
72+
void enumerationToEnd() const {
73+
std::istringstream istr("void a(){} void main(){ if(true){a();} }");
74+
TokenList tokenList(nullptr);
75+
tokenList.createTokens(istr, "test.cpp");
76+
ASSERT_EQUALS("", testTokenRange(ConstTokenRange{ tokenList.front(), nullptr }, tokenList.front(), nullptr));
77+
}
78+
79+
void untilHelperToEnd() const {
80+
std::istringstream istr("void a(){} void main(){ if(true){a();} }");
81+
TokenList tokenList(nullptr);
82+
tokenList.createTokens(istr, "test.cpp");
83+
ASSERT_EQUALS("", testTokenRange(tokenList.front()->until(nullptr), tokenList.front(), nullptr));
84+
}
85+
86+
void untilHelperPartWay() const {
87+
std::istringstream istr("void a(){} void main(){ if(true){a();} }");
88+
TokenList tokenList(nullptr);
89+
tokenList.createTokens(istr, "test.cpp");
90+
const Token* start = tokenList.front()->tokAt(4);
91+
const Token* end = start->tokAt(8);
92+
ASSERT_EQUALS("", testTokenRange(start->until(end), start, end));
93+
}
94+
95+
void partialEnumeration() const {
96+
std::istringstream istr("void a(){} void main(){ if(true){a();} }");
97+
TokenList tokenList(nullptr);
98+
tokenList.createTokens(istr, "test.cpp");
99+
const Token* start = tokenList.front()->tokAt(4);
100+
const Token* end = tokenList.front()->tokAt(10);
101+
ASSERT_EQUALS("", testTokenRange(ConstTokenRange{ start, end }, start, end));
102+
}
103+
104+
void scopeExample() const {
105+
Settings settings;
106+
Tokenizer tokenizer{ &settings, nullptr };
107+
std::istringstream sample("void a(){} void main(){ if(true){a();} }");
108+
tokenizer.tokenize(sample, "test.cpp");
109+
110+
const SymbolDatabase* sd = tokenizer.getSymbolDatabase();
111+
const Scope& scope = *std::next(sd->scopeList.begin(), 3); //The scope of the if block
112+
113+
std::ostringstream contents;
114+
for (const Token* t : ConstTokenRange{ scope.bodyStart->next(), scope.bodyEnd })
115+
{
116+
contents << t->str();
117+
}
118+
ASSERT_EQUALS("a();", contents.str());
119+
}
120+
121+
void exampleAlgorithms() const {
122+
std::istringstream istr("void a(){} void main(){ if(true){a();} }");
123+
TokenList tokenList(nullptr);
124+
tokenList.createTokens(istr, "test.cpp");
125+
ConstTokenRange range{ tokenList.front(), nullptr };
126+
ASSERT_EQUALS(true, std::all_of(range.begin(), range.end(), [](const Token*) {return true;}));
127+
ASSERT_EQUALS(true, std::any_of(range.begin(), range.end(), [](const Token* t) {return t->str() == "true";}));
128+
ASSERT_EQUALS("true", (*std::find_if(range.begin(), range.end(), [](const Token* t) {return t->str() == "true";}))->str());
129+
ASSERT_EQUALS(3, std::count_if(range.begin(), range.end(), [](const Token* t) {return t->str() == "{";}));
130+
}
131+
};
132+
133+
REGISTER_TEST(TestTokenRange)

0 commit comments

Comments
 (0)