Optimization: Reduced peak memory usage (30% in my test case) by immediately deleting simplecpp::TokenList while creating the cppcheck TokenList.

PKEuS · PKEuS · commit c9d8f607df84 · 2020-05-19T12:08:17.000+02:00
diff --git a/lib/cppcheck.cpp b/lib/cppcheck.cpp
@@ -713,7 +713,7 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
                 {
                     Timer timer("Tokenizer::createTokens", mSettings.showtime, &s_timerResults);
                     simplecpp::TokenList tokensP = preprocessor.preprocess(tokens1, mCurrentConfig, files, true);
-                    mTokenizer.createTokens(&tokensP);
+                    mTokenizer.createTokens(std::move(tokensP));
                 }
                 hasValidConfig = true;
 
diff --git a/lib/tokenize.cpp b/lib/tokenize.cpp
@@ -2302,11 +2302,11 @@ bool Tokenizer::createTokens(std::istream &code,
     return list.createTokens(code, FileName);
 }
 
-void Tokenizer::createTokens(const simplecpp::TokenList *tokenList)
+void Tokenizer::createTokens(simplecpp::TokenList&& tokenList)
 {
     // make sure settings specified
     assert(mSettings);
-    list.createTokens(tokenList);
+    list.createTokens(std::move(tokenList));
 }
 
 bool Tokenizer::simplifyTokens1(const std::string &configuration)
diff --git a/lib/tokenize.h b/lib/tokenize.h
@@ -110,7 +110,7 @@ class CPPCHECKLIB Tokenizer {
     bool isScopeNoReturn(const Token *endScopeToken, bool *unknown = nullptr) const;
 
     bool createTokens(std::istream &code, const std::string& FileName);
-    void createTokens(const simplecpp::TokenList *tokenList);
+    void createTokens(simplecpp::TokenList&& tokenList);
 
     bool simplifyTokens1(const std::string &configuration);
     /**
diff --git a/lib/tokenlist.cpp b/lib/tokenlist.cpp
@@ -377,17 +377,17 @@ bool TokenList::createTokens(std::istream &code, const std::string& file0)
     simplecpp::OutputList outputList;
     simplecpp::TokenList tokens(code, mFiles, file0, &outputList);
 
-    createTokens(&tokens);
+    createTokens(std::move(tokens));
 
     return outputList.empty();
 }
 
 //---------------------------------------------------------------------------
 
-void TokenList::createTokens(const simplecpp::TokenList *tokenList)
+void TokenList::createTokens(simplecpp::TokenList&& tokenList)
 {
-    if (tokenList->cfront())
-        mOrigFiles = mFiles = tokenList->cfront()->location.files;
+    if (tokenList.cfront())
+        mOrigFiles = mFiles = tokenList.cfront()->location.files;
     else
         mFiles.clear();
 
@@ -401,7 +401,7 @@ void TokenList::createTokens(const simplecpp::TokenList *tokenList)
         mIsCpp = (mSettings->enforcedLang == Settings::CPP);
     }
 
-    for (const simplecpp::Token *tok = tokenList->cfront(); tok; tok = tok->next) {
+    for (const simplecpp::Token *tok = tokenList.cfront(); tok;) {
 
         std::string str = tok->str();
 
@@ -423,6 +423,10 @@ void TokenList::createTokens(const simplecpp::TokenList *tokenList)
         mTokensFrontBack.back->linenr(tok->location.line);
         mTokensFrontBack.back->column(tok->location.col);
         mTokensFrontBack.back->isExpandedMacro(!tok->macro.empty());
+
+        tok = tok->next;
+        if (tok)
+            tokenList.deleteToken(tok->previous);
     }
 
     if (mSettings && mSettings->relativePaths) {
diff --git a/lib/tokenlist.h b/lib/tokenlist.h
@@ -98,7 +98,7 @@ class CPPCHECKLIB TokenList {
      */
     bool createTokens(std::istream &code, const std::string& file0 = emptyString);
 
-    void createTokens(const simplecpp::TokenList *tokenList);
+    void createTokens(simplecpp::TokenList&& tokenList);
 
     /** Deallocate list */
     void deallocateTokens();
diff --git a/test/testcondition.cpp b/test/testcondition.cpp
@@ -138,7 +138,7 @@ class TestCondition : public TestFixture {
 
         // Tokenizer..
         Tokenizer tokenizer(&settings0, this);
-        tokenizer.createTokens(&tokens2);
+        tokenizer.createTokens(std::move(tokens2));
         tokenizer.simplifyTokens1("");
 
         // Run checks..
diff --git a/test/testincompletestatement.cpp b/test/testincompletestatement.cpp
@@ -51,7 +51,7 @@ class TestIncompleteStatement : public TestFixture {
 
         // Tokenize..
         Tokenizer tokenizer(&settings, this);
-        tokenizer.createTokens(&tokens2);
+        tokenizer.createTokens(std::move(tokens2));
         tokenizer.simplifyTokens1("");
 
         // Check for incomplete statements..
diff --git a/test/testleakautovar.cpp b/test/testleakautovar.cpp
@@ -218,7 +218,7 @@ class TestLeakAutoVar : public TestFixture {
 
         // Tokenizer..
         Tokenizer tokenizer(&settings, this);
-        tokenizer.createTokens(&tokens2);
+        tokenizer.createTokens(std::move(tokens2));
         tokenizer.simplifyTokens1("");
 
         // Check for leaks..
diff --git a/test/testnullpointer.cpp b/test/testnullpointer.cpp
@@ -163,7 +163,7 @@ class TestNullPointer : public TestFixture {
 
         // Tokenizer..
         Tokenizer tokenizer(&settings, this);
-        tokenizer.createTokens(&tokens2);
+        tokenizer.createTokens(std::move(tokens2));
         tokenizer.simplifyTokens1("");
 
         // Check for null pointer dereferences..
diff --git a/test/testother.cpp b/test/testother.cpp
@@ -300,7 +300,7 @@ class TestOther : public TestFixture {
 
         // Tokenizer..
         Tokenizer tokenizer(settings, this);
-        tokenizer.createTokens(&tokens2);
+        tokenizer.createTokens(std::move(tokens2));
         tokenizer.simplifyTokens1("");
 
         // Check..
diff --git a/test/testsizeof.cpp b/test/testsizeof.cpp
@@ -81,7 +81,7 @@ class TestSizeof : public TestFixture {
 
         // Tokenize..
         Tokenizer tokenizer(&settings, this);
-        tokenizer.createTokens(&tokens2);
+        tokenizer.createTokens(std::move(tokens2));
         tokenizer.simplifyTokens1("");
 
         // Check...
diff --git a/test/testunusedprivfunc.cpp b/test/testunusedprivfunc.cpp
@@ -103,7 +103,7 @@ class TestUnusedPrivateFunction : public TestFixture {
 
         // Tokenize..
         Tokenizer tokenizer(&settings, this);
-        tokenizer.createTokens(&tokens2);
+        tokenizer.createTokens(std::move(tokens2));
         tokenizer.simplifyTokens1("");
 
         // Check for unused private functions..
diff --git a/test/testvalueflow.cpp b/test/testvalueflow.cpp
@@ -307,7 +307,7 @@ class TestValueFlow : public TestFixture {
 
         // Tokenize..
         Tokenizer tokenizer(&settings, this);
-        tokenizer.createTokens(&tokens2);
+        tokenizer.createTokens(std::move(tokens2));
         tokenizer.simplifyTokens1("");
 
         settings.debugwarnings = false;

Original file line number	Diff line number	Diff line change
`@@ -713,7 +713,7 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string`
`713`	`713`	`{`
`714`	`714`	`Timer timer("Tokenizer::createTokens", mSettings.showtime, &s_timerResults);`
`715`	`715`	`simplecpp::TokenList tokensP = preprocessor.preprocess(tokens1, mCurrentConfig, files, true);`
`716`		`- mTokenizer.createTokens(&tokensP);`
	`716`	`+ mTokenizer.createTokens(std::move(tokensP));`
`717`	`717`	`}`
`718`	`718`	`hasValidConfig = true;`
`719`	`719`
Original file line number	Diff line number	Diff line change
`@@ -2302,11 +2302,11 @@ bool Tokenizer::createTokens(std::istream &code,`
`2302`	`2302`	`return list.createTokens(code, FileName);`
`2303`	`2303`	`}`
`2304`	`2304`
`2305`		`-void Tokenizer::createTokens(const simplecpp::TokenList *tokenList)`
	`2305`	`+void Tokenizer::createTokens(simplecpp::TokenList&& tokenList)`
`2306`	`2306`	`{`
`2307`	`2307`	`// make sure settings specified`
`2308`	`2308`	`assert(mSettings);`
`2309`		`- list.createTokens(tokenList);`
	`2309`	`+ list.createTokens(std::move(tokenList));`
`2310`	`2310`	`}`
`2311`	`2311`
`2312`	`2312`	`bool Tokenizer::simplifyTokens1(const std::string &configuration)`