fixed things related to automatic benchmarking

hypro · Jun 7, 2019 · 2d918ee · 2d918ee
1 parent 4e688b3
commit 2d918ee
Show file tree

Hide file tree

Showing 19 changed files with 122 additions and 106 deletions.
diff --git a/src/test/benchmark/hyproBenchmark/Settings.h b/src/test/benchmark/hyproBenchmark/Settings.h
@@ -12,11 +12,12 @@ namespace benchmark
     struct Settings
     {
         static constexpr std::size_t maxDimension = 100;        // maximal dimension for benchmark objects
-        static constexpr std::size_t iterations = 100;        // number of iterations per benchmark
+        static constexpr std::size_t iterations = 10;        // number of iterations per benchmark
         static constexpr std::size_t multiplications = 100;
         static constexpr std::size_t evaluations = 100;
         static constexpr std::size_t stepSize = 5;
         static constexpr std::size_t stepSizeEvaluations = 10;
         static constexpr std::size_t stepSizeMultiplications = 2;
+        static constexpr std::size_t timeoutSec = 60; // if an operation takes longer than this, stop computing it from now on.
     };
 } // benchmark
diff --git a/src/test/benchmark/hyproBenchmark/box/affineTransformation.cpp b/src/test/benchmark/hyproBenchmark/box/affineTransformation.cpp
@@ -14,64 +14,79 @@ namespace box {
         box.insert(carl::Interval<::benchmark::Number>(-1,1));
 
         // initialize random number generator
-        mt19937 generator;
+        std::mt19937 generator;
         std::uniform_int_distribution<int> dist = std::uniform_int_distribution<int>(0,10);
 
+        bool timeout = false;
+        bool timeoutNaive = false;
+        bool timeoutPpl = false;
+
         // iterate over dimensions
         for(std::size_t d = 1; d < settings.maxDimension; ++d) {
             // create instances
             std::vector<hypro::matrix_t<::benchmark::Number>> matrices;
             std::vector<hypro::vector_t<::benchmark::Number>> vectors;
-            Timer creationTimer;
-            for(std::size_t i = 0; i < settings.iterations; ++i) {
-                hypro::matrix_t<::benchmark::Number> matrix = hypro::matrix_t<::benchmark::Number>(d,d);
-                hypro::vector_t<::benchmark::Number> vector = hypro::vector_t<::benchmark::Number>(d);
-                for(std::size_t row = 0; row < d; ++row) {
-                    for(std::size_t col = 0; col < d; ++col) {
-                        matrix(row,col) = dist(generator);
+
+            if(!timeout) {
+                Timer creationTimer;
+                for(std::size_t i = 0; i < settings.iterations; ++i) {
+                    hypro::matrix_t<::benchmark::Number> matrix = hypro::matrix_t<::benchmark::Number>(d,d);
+                    hypro::vector_t<::benchmark::Number> vector = hypro::vector_t<::benchmark::Number>(d);
+                    for(std::size_t row = 0; row < d; ++row) {
+                        for(std::size_t col = 0; col < d; ++col) {
+                            matrix(row,col) = dist(generator);
+                        }
+                        vector(row) = dist(generator);
                     }
-                    vector(row) = dist(generator);
+                    matrices.emplace_back(std::move(matrix));
+                    vectors.emplace_back(std::move(vector));
                 }
-                matrices.emplace_back(std::move(matrix));
-                vectors.emplace_back(std::move(vector));
-            }
-            auto creationTime = creationTimer.elapsed();
-            //std::cout << "Dimension " << d << ": Creation took " << creationTime.count() << " sec." << std::endl;
-            ress.mCreationTime += creationTime;
+                auto creationTime = creationTimer.elapsed();
+                //std::cout << "Dimension " << d << ": Creation took " << creationTime.count() << " sec." << std::endl;
+                ress.mCreationTime += creationTime;
 
-            // run instances
-            Timer runTimerHyPro;
-            for(std::size_t i = 0; i < settings.iterations; ++i) {
-                box.affineTransformation(matrices[i], vectors[i]);
+                // run instances
+                Timer runTimerHyPro;
+                for(std::size_t i = 0; i < settings.iterations; ++i) {
+                    box.affineTransformation(matrices[i], vectors[i]);
+                }
+                auto runningTime = runTimerHyPro.elapsed();
+                ress.emplace_back({"affineTransformation",runningTime/settings.iterations,static_cast<int>(d)});
+                std::cout << "Dimension " << d << ":  Running took " << runningTime.count() << " sec." << std::endl;
+                if(runningTime.count() > settings.timeoutSec) {
+                    timeout = true;
+                }
+                ress.mRunningTime += runningTime;
             }
-            auto runningTime = runTimerHyPro.elapsed();
-            ress.emplace_back({"affineTransformation",runningTime/settings.iterations,static_cast<int>(d)});
-            std::cout << "Dimension " << d << ":  Running took " << runningTime.count() << " sec." << std::endl;
 
-            Timer runTimerHyProNaive;
-            for(std::size_t i = 0; i < settings.iterations; ++i) {
-              std::vector<hypro::Point<::benchmark::Number>> vertices = box.vertices();
-            	hypro::Point<::benchmark::Number> manualMin = hypro::Point<::benchmark::Number>(matrices[i]*(vertices.begin()->rawCoordinates()));
-            	hypro::Point<::benchmark::Number> manualMax = hypro::Point<::benchmark::Number>(matrices[i]*(vertices.begin()->rawCoordinates()));
-            	for(const auto& v : vertices) {
-            		hypro::Point<::benchmark::Number> t = hypro::Point<::benchmark::Number>(matrices[i]*v.rawCoordinates());
-            		for(std::size_t d = 0; d < box.dimension(); ++d) {
-            			if(manualMin.at(d) > t.at(d)) {
-            				manualMin[d] = t[d];
-            			}
-            			if(manualMax.at(d) < t.at(d)) {
-            				manualMax[d] = t[d];
-            			}
-            		}
-            	}
-              manualMin += vectors[i];
-              manualMax += vectors[i];
-            }
-            auto runningTimeNaive = runTimerHyProNaive.elapsed();
-            ress.emplace_back({"affineTransformationNaive",runningTimeNaive/settings.iterations,static_cast<int>(d)});
-            std::cout << "Dimension " << d << ":  Running took " << runningTimeNaive.count() << " sec." << std::endl;
+            if(!timeoutNaive) {
+                Timer runTimerHyProNaive;
+                for(std::size_t i = 0; i < settings.iterations; ++i) {
+                std::vector<hypro::Point<::benchmark::Number>> vertices = box.vertices();
+                    hypro::Point<::benchmark::Number> manualMin = hypro::Point<::benchmark::Number>(matrices[i]*(vertices.begin()->rawCoordinates()));
+                    hypro::Point<::benchmark::Number> manualMax = hypro::Point<::benchmark::Number>(matrices[i]*(vertices.begin()->rawCoordinates()));
+                    for(const auto& v : vertices) {
+                        hypro::Point<::benchmark::Number> t = hypro::Point<::benchmark::Number>(matrices[i]*v.rawCoordinates());
+                        for(std::size_t d = 0; d < box.dimension(); ++d) {
+                            if(manualMin.at(d) > t.at(d)) {
+                                manualMin[d] = t[d];
+                            }
+                            if(manualMax.at(d) < t.at(d)) {
+                                manualMax[d] = t[d];
+                            }
+                        }
+                    }
+                manualMin += vectors[i];
+                manualMax += vectors[i];
+                }
+                auto runningTimeNaive = runTimerHyProNaive.elapsed();
+                ress.emplace_back({"affineTransformationNaive",runningTimeNaive/settings.iterations,static_cast<int>(d)});
+                std::cout << "Dimension " << d << ":  Running took " << runningTimeNaive.count() << " sec." << std::endl;
 
-            ress.mRunningTime += runningTime;
+                if(runningTimeNaive.count() > settings.timeoutSec) {
+                    timeoutNaive = true;
+                }
+            }
 
             // prepare next run
             box.insert(carl::Interval<::benchmark::Number>(-1,1));

diff --git a/src/test/benchmark/hyproBenchmark/box/benchmarkBox.cpp b/src/test/benchmark/hyproBenchmark/box/benchmarkBox.cpp
@@ -9,41 +9,41 @@ namespace box
         Results<std::size_t> ress;
         std::vector<std::future<Results<std::size_t>>> workingTasks;
 
-        //workingTasks.emplace_back(std::async(std::launch::deferred,intersectHalfspace,settings));
+        workingTasks.emplace_back(std::async(std::launch::deferred,intersectHalfspace,settings));
         workingTasks.emplace_back(std::async(std::launch::deferred,affineTransformation,settings));
-        //workingTasks.emplace_back(std::async(std::launch::deferred,unite,settings));
-        //workingTasks.emplace_back(std::async(std::launch::deferred,intersect,settings));
-        //workingTasks.emplace_back(std::async(std::launch::deferred,computeSupport,settings));
+        workingTasks.emplace_back(std::async(std::launch::deferred,unite,settings));
+        workingTasks.emplace_back(std::async(std::launch::deferred,intersect,settings));
+        workingTasks.emplace_back(std::async(std::launch::deferred,computeSupport,settings));
 
         // half-space intersection
-        //auto tmp = workingTasks[0].get();
-        //tmp.createCSV("boxIntersectHalfspace", "\t", "intersectHalfspace");
-        //tmp.createCSV("boxIntersectHalfspaceNaive", "\t", "intersectHalfspaceNaive");
-        //tmp.createCSV("boxIntersectHalfspacePPL", "\t", "intersectHalfspacePPL");
-        //std::cout << "Benchmarked hsp intersection." << std::endl;
+        auto tmp = workingTasks[0].get();
+        tmp.createCSV("boxIntersectHalfspace", "\t", "intersectHalfspace");
+        tmp.createCSV("boxIntersectHalfspaceNaive", "\t", "intersectHalfspaceNaive");
+        tmp.createCSV("boxIntersectHalfspacePPL", "\t", "intersectHalfspacePPL");
+        std::cout << "Benchmarked hsp intersection." << std::endl;
 
         // affine transformation
-        auto tmp = workingTasks[0].get();
+        tmp = workingTasks[1].get();
         tmp.createCSV("boxAffineTransformation", "\t", "affineTransformation");
         tmp.createCSV("boxAffineTransformationNaive", "\t", "affineTransformationNaive");
         std::cout << "Benchmarked affine transformation." << std::endl;
 
         // union
-        //tmp = workingTasks[2].get();
-        //tmp.createCSV("boxUnion", "\t");
-        //std::cout << "Benchmarked union." << std::endl;
+        tmp = workingTasks[2].get();
+        tmp.createCSV("boxUnion", "\t");
+        std::cout << "Benchmarked union." << std::endl;
 
         // intersection
-        //tmp = workingTasks[3].get();
-        //tmp.createCSV("boxIntersection", "\t","intersect");
-        //tmp.createCSV("boxIntersectionPPL", "\t", "intersectPPL");
-        //std::cout << "Benchmarked intersection." << std::endl;
+        tmp = workingTasks[3].get();
+        tmp.createCSV("boxIntersection", "\t","intersect");
+        tmp.createCSV("boxIntersectionPPL", "\t", "intersectPPL");
+        std::cout << "Benchmarked intersection." << std::endl;
 
         // support
-        //auto tmp = workingTasks[0].get();
-        //tmp.createCSV("boxComputeSupport", "\t", "computeSupport");
-        //tmp.createCSV("boxComputeSupportNaive", "\t", "computeSupportNaive");
-        //std::cout << "Benchmarked compute support." << std::endl;
+        tmp = workingTasks[4].get();
+        tmp.createCSV("boxComputeSupport", "\t", "computeSupport");
+        tmp.createCSV("boxComputeSupportNaive", "\t", "computeSupportNaive");
+        std::cout << "Benchmarked compute support." << std::endl;
 
         return ress;
     }

diff --git a/src/test/benchmark/hyproBenchmark/box/computeSupport.cpp b/src/test/benchmark/hyproBenchmark/box/computeSupport.cpp
@@ -9,7 +9,7 @@ namespace box {
         box.insert(carl::Interval<::benchmark::Number>(-1,1));
 
         // initialize random number generator
-        mt19937 generator;
+        std::mt19937 generator;
         std::uniform_int_distribution<int> dist = std::uniform_int_distribution<int>(0,10);
 
         // iterate over dimensions

diff --git a/src/test/benchmark/hyproBenchmark/box/intersect.cpp b/src/test/benchmark/hyproBenchmark/box/intersect.cpp
@@ -12,7 +12,7 @@ namespace box {
         #endif
 
         // initialize random number generator
-        mt19937 generator;
+        std::mt19937 generator;
         std::uniform_int_distribution<int> dist = std::uniform_int_distribution<int>(0,10);
 
         // iterate over dimensions
@@ -82,7 +82,7 @@ namespace box {
                 pplLhsBoxes[i].intersection_assign(pplRhsBoxes[i]);
             }
             runningTime = runTimerPPL.elapsed();
-            ress.emplace_back({"intersectPPL",runningTime/settings.iterations,d});
+            ress.emplace_back({"intersectPPL",runningTime/settings.iterations,static_cast<int>(d)});
             std::cout << "Dimension " << d << ":  Running took " << runningTime.count() << " sec (PPL)." << std::endl;
             #endif
 

diff --git a/src/test/benchmark/hyproBenchmark/box/intersectHalfspace.cpp b/src/test/benchmark/hyproBenchmark/box/intersectHalfspace.cpp
@@ -14,7 +14,7 @@ namespace box {
         box.insert(carl::Interval<::benchmark::Number>(-1,1));
 
         // initialize random number generator
-        mt19937 generator;
+        std::mt19937 generator;
         std::uniform_int_distribution<int> dist = std::uniform_int_distribution<int>(0,10);
 
         // iterate over dimensions
@@ -35,8 +35,8 @@ namespace box {
                 #ifdef HYPRO_USE_PPL
                 Parma_Polyhedra_Library::Constraint c;
                 Parma_Polyhedra_Library::Linear_Expression e;
-                for (dimension_type i = 0; i < d; ++i )
-                    e += hsps.back().normal()(i) * Variable(i);
+                for (Parma_Polyhedra_Library::dimension_type i = 0; i < d; ++i )
+                    e += hsps.back().normal()(i) * Parma_Polyhedra_Library::Variable(i);
                 e += -hsps.back().offset();
                 c = e <= 0;
                 pplHsps.emplace_back(c);
@@ -78,7 +78,7 @@ namespace box {
                 b.refine_with_constraint(pplHsps[i]);
                 pplRT += runTimerPPL.elapsed();
             }
-            ress.emplace_back({"intersectHalfspacePPL",pplRT/settings.iterations,d});
+            ress.emplace_back({"intersectHalfspacePPL",pplRT/settings.iterations,static_cast<int>(d)});
             std::cout << "Dimension " << d << ":  Running took " << pplRT.count() << " sec (PPL)." << std::endl;
             #endif
 

diff --git a/src/test/benchmark/hyproBenchmark/box/satisfiesHalfspace.cpp b/src/test/benchmark/hyproBenchmark/box/satisfiesHalfspace.cpp
@@ -14,7 +14,7 @@ namespace box {
         box.insert(carl::Interval<::benchmark::Number>(-1,1));
 
         // initialize random number generator
-        mt19937 generator;
+        std::mt19937 generator;
         std::uniform_int_distribution<int> dist = std::uniform_int_distribution<int>(0,10);
 
         // iterate over dimensions
@@ -35,8 +35,8 @@ namespace box {
                 #ifdef HYPRO_USE_PPL
                 Parma_Polyhedra_Library::Constraint c;
                 Parma_Polyhedra_Library::Linear_Expression e;
-                for (dimension_type i = 0; i < d; ++i )
-                    e += hsps.back().normal()(i) * Variable(i);
+                for (Parma_Polyhedra_Library::dimension_type i = 0; i < d; ++i )
+                    e += hsps.back().normal()(i) * Parma_Polyhedra_Library::Variable(i);
                 e += -hsps.back().offset();
                 c = e <= 0;
                 pplHsps.emplace_back(c);
@@ -82,4 +82,3 @@ namespace box {
 
 } // box
 } // benchmark
-
diff --git a/src/test/benchmark/hyproBenchmark/box/unite.cpp b/src/test/benchmark/hyproBenchmark/box/unite.cpp
@@ -12,7 +12,7 @@ namespace box {
         #endif
 
         // initialize random number generator
-        mt19937 generator;
+        std::mt19937 generator;
         std::uniform_int_distribution<int> dist = std::uniform_int_distribution<int>(0,10);
 
         // iterate over dimensions

diff --git a/src/test/benchmark/hyproBenchmark/polytopes/affineTransformation.cpp b/src/test/benchmark/hyproBenchmark/polytopes/affineTransformation.cpp
@@ -13,7 +13,7 @@ namespace polytope {
         #endif
 
         // initialize random number generator
-        mt19937 generator;
+        std::mt19937 generator;
         std::uniform_int_distribution<int> dist = std::uniform_int_distribution<int>(0,10);
 
         // iterate over dimensions

diff --git a/src/test/benchmark/hyproBenchmark/polytopes/benchmarkPolytope.cpp b/src/test/benchmark/hyproBenchmark/polytopes/benchmarkPolytope.cpp
@@ -28,11 +28,11 @@ namespace polytope
         //ress.insert(ress.end(), tmp.begin(), tmp.end());
 
         // intersection
-        //std::cout << "Benchmarking intersection." << std::endl;
-        //tmp = intersect(settings);
-        //tmp.createCSV("polytopeIntersection", "\t","intersect");
-        //tmp.createCSV("polytopeIntersectionPPL", "\t", "intersectPPL");
-        //ress.insert(ress.end(), tmp.begin(), tmp.end());
+        std::cout << "Benchmarking intersection." << std::endl;
+        tmp = intersect(settings);
+        tmp.createCSV("polytopeIntersection", "\t","intersect");
+        tmp.createCSV("polytopeIntersectionPPL", "\t", "intersectPPL");
+        ress.insert(ress.end(), tmp.begin(), tmp.end());
 
         std::cout << "Polytope benchmarking took " << totalRunningTime.elapsedMs() << " sec in total." << std::endl;
 

diff --git a/src/test/benchmark/hyproBenchmark/polytopes/intersect.cpp b/src/test/benchmark/hyproBenchmark/polytopes/intersect.cpp
@@ -13,7 +13,7 @@ namespace polytope {
         #endif
 
         // initialize random number generator
-        mt19937 generator;
+        std::mt19937 generator;
         std::uniform_int_distribution<int> dist = std::uniform_int_distribution<int>(0,10);
 
         // iterate over dimensions
@@ -83,7 +83,7 @@ namespace polytope {
                 pplLhsBoxes[i].intersection_assign(pplRhsBoxes[i]);
             }
             runningTime = runTimerPPL.elapsed();
-            ress.emplace_back({"intersectPPL",runningTime/settings.iterations,d});
+            ress.emplace_back({"intersectPPL",runningTime/settings.iterations,static_cast<int>(d)});
             std::cout << "Dimension " << d << ":  Running took " << runningTime.count() << " sec (PPL)." << std::endl;
             #endif
 

diff --git a/src/test/benchmark/hyproBenchmark/runBenchmarks.cpp b/src/test/benchmark/hyproBenchmark/runBenchmarks.cpp
@@ -28,7 +28,8 @@ int main(int argc, char const *argv[])
     //std::cout << "Benchmark polytopes." << std::endl;
     //benchmark::polytope::run(s);
 
-    //std::cout << "Benchmark support functions." << std::endl;
+    std::cout << "Benchmark support functions." << std::endl;
+    benchmark::sf::run(s);
     //std::async(std::launch::async,benchmark::sf::run,s);
 
     std::cout << "Benchmarking took " << general.elapsedMs() << " sec." << std::endl;

diff --git a/src/test/benchmark/hyproBenchmark/supportFunction/affineTransformation.cpp b/src/test/benchmark/hyproBenchmark/supportFunction/affineTransformation.cpp
@@ -7,7 +7,7 @@ namespace sf {
         Results<std::size_t> ress;
 
         // initialize random number generator
-        mt19937 generator;
+        std::mt19937 generator;
         std::uniform_int_distribution<int> dist = std::uniform_int_distribution<int>(0,10);
 
         // iterate over dimensions

diff --git a/src/test/benchmark/hyproBenchmark/supportFunction/affineTransformationEvaluation.cpp b/src/test/benchmark/hyproBenchmark/supportFunction/affineTransformationEvaluation.cpp
@@ -6,10 +6,10 @@ namespace sf {
     void affineTrafoCreator(std::map<std::size_t, std::vector<hypro::matrix_t<::benchmark::Number>>>& matricesMap,
                             std::map<std::size_t, std::vector<hypro::vector_t<::benchmark::Number>>>& vectorsMap,
                             std::map<std::size_t, std::vector<hypro::vector_t<::benchmark::Number>>>& directionsMap,
-                            const Settings& settings) 
+                            const Settings& settings)
     {
         // initialize random number generator
-        mt19937 generator;
+        std::mt19937 generator;
         std::uniform_int_distribution<int> dist = std::uniform_int_distribution<int>(0,10);
 
         for(std::size_t d = 1; d < settings.maxDimension; ++d) {
@@ -40,7 +40,7 @@ namespace sf {
 
     Results<std::size_t> affineTransformationEvaluation(const Settings& settings) {
         Results<std::size_t> ress;
-        
+
         std::map<std::size_t, std::vector<hypro::matrix_t<::benchmark::Number>>> matricesMap;
         std::map<std::size_t, std::vector<hypro::vector_t<::benchmark::Number>>> vectorsMap;
         std::map<std::size_t, std::vector<hypro::vector_t<::benchmark::Number>>> directionsMap;
@@ -60,7 +60,7 @@ namespace sf {
             }
             hypro::SupportFunction<::benchmark::Number> sFunct(constraints,constants);
             hypro::SupportFunctionT<::benchmark::Number,hypro::Converter<::benchmark::Number>,hypro::NoTrafoReduction> sFunctNoReduction(constraints,constants);
-            
+
             // run instances asynchronous
 
             auto& mVector = matricesMap[d];