Skip to content

Commit

Permalink
Port libFuzzer tests to LIT. Do not require two-stage build for check…
Browse files Browse the repository at this point in the history 
…-fuzzer.

This revision ports all libFuzzer tests apart from the unittest to LIT.
The advantages of doing so include:

 - Tests being self-contained
 - Much easier debugging of a single test
 - No need for using a two-stage compilation

The unit-test is still compiled using CMake, but it does not need a
freshly built compiler.

NOTE: The previous two-stage bot configuration will NOT work, as in the
second stage build LLVM_USE_SANITIZER is set, which disables ASAN from
being built.
Thus bots will be reconfigured in the next few commits.

Differential Revision: https://reviews.llvm.org/D36295

llvm-svn: 310075
  • Loading branch information
George Karpenkov committed Aug 4, 2017
1 parent ff77cc7 commit 8ecdd7b
Show file tree
Hide file tree
Showing 73 changed files with 347 additions and 445 deletions.
9 changes: 6 additions & 3 deletions llvm/docs/LibFuzzer.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -624,12 +624,15 @@ you will eventually run out of RAM (see the ``-rss_limit_mb`` flag).
Developing libFuzzer
====================

Building libFuzzer as a part of LLVM project and running its test requires
fresh clang as the host compiler and special CMake configuration:
LibFuzzer is built as a part of LLVM project by default on macos and Linux.
Users of other operating systems can explicitly request compilation using
``-DLIBFUZZER_ENABLE=YES`` flag.
Tests are run using ``check-fuzzer`` target from the build directory
(note that tests will take a long time to run if the compiler was compiled
without optimizations):

.. code-block:: console
cmake -GNinja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_USE_SANITIZER=Address -DLLVM_USE_SANITIZE_COVERAGE=YES -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON /path/to/llvm
ninja check-fuzzer
Expand Down
21 changes: 8 additions & 13 deletions llvm/lib/Fuzzer/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,22 +13,17 @@ if( APPLE )
endif()
endif()

set(LIBFUZZER_FLAGS_BASE "${CMAKE_CXX_FLAGS}")
if( LLVM_USE_SANITIZE_COVERAGE )
if(NOT "${LLVM_USE_SANITIZER}" STREQUAL "Address")
message(FATAL_ERROR
"LibFuzzer and its tests require LLVM_USE_SANITIZER=Address and "
"LLVM_USE_SANITIZE_COVERAGE=YES to be set."
)
endif()

# Disable the coverage and sanitizer instrumentation for the fuzzer itself.
set(CMAKE_CXX_FLAGS "${LIBFUZZER_FLAGS_BASE} -fno-sanitize-coverage=trace-pc-guard,edge,trace-cmp,indirect-calls,8bit-counters -Werror")
if (CMAKE_SYSTEM_NAME MATCHES "Darwin|Linux")
set(LIBFUZZER_ENABLED_CHECK ON)
else()
set(LIBFUZZER_ENABLED_CHECK OFF)
endif()

# Compile libFuzzer if the compilation is specifically requested, OR
# if the platform is known to be working.
if ( LLVM_USE_SANITIZE_COVERAGE OR CMAKE_SYSTEM_NAME MATCHES "Darwin|Linux" )
set(LIBFUZZER_ENABLE ${LIBFUZZER_ENABLED_CHECK} CACHE BOOL "Build libFuzzer and its tests")

if (LIBFUZZER_ENABLE)
add_library(LLVMFuzzerNoMainObjects OBJECT
FuzzerCrossOver.cpp
FuzzerDriver.cpp
Expand Down Expand Up @@ -70,7 +65,7 @@ if (MSVC)
add_custom_command(TARGET check-fuzzer
COMMAND cmake -E echo "check-fuzzer is disalbed on Windows")
else()
if( LLVM_USE_SANITIZE_COVERAGE AND LLVM_INCLUDE_TESTS )
if (LLVM_INCLUDE_TESTS AND LIBFUZZER_ENABLE)
add_subdirectory(test)
endif()
endif()
213 changes: 18 additions & 195 deletions llvm/lib/Fuzzer/test/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,3 @@
# Build all these tests with -O0, otherwise optimizations may merge some
# basic blocks and we'll fail to discover the targets.
# We change the flags for every build type because we might be doing
# a multi-configuration build (e.g. Xcode) where CMAKE_BUILD_TYPE doesn't
# mean anything.
set(variables_to_filter
CMAKE_CXX_FLAGS_RELEASE
CMAKE_CXX_FLAGS_DEBUG
CMAKE_CXX_FLAGS_RELWITHDEBINFO
CMAKE_CXX_FLAGS_MINSIZEREL
LIBFUZZER_FLAGS_BASE
)
foreach (VARNAME ${variables_to_filter})
string(REGEX REPLACE "([-/]O)[123s]" "\\10" ${VARNAME} "${${VARNAME}}")
endforeach()

# Enable the coverage instrumentation (it is disabled for the Fuzzer lib).
set(CMAKE_CXX_FLAGS "${LIBFUZZER_FLAGS_BASE} -fsanitize-coverage=trace-pc-guard,indirect-calls,trace-cmp,trace-div,trace-gep -gline-tables-only")

if(MSVC)
# For tests use the CRT specified for release build
# (asan doesn't support MDd and MTd)
Expand All @@ -39,105 +20,6 @@ if(MSVC)
set(CMAKE_CXX_CREATE_SHARED_LIBRARY "<CMAKE_CXX_COMPILER> ${CMAKE_CXX_FLAGS} ${CRT_FLAG} /LD <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG> <TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES> /link <LINK_FLAGS>")
endif()

add_custom_target(TestBinaries)

# add_libfuzzer_test(<name>
# SOURCES source0.cpp [source1.cpp ...]
# )
#
# Declares a LibFuzzer test executable with target name LLVMFuzzer-<name>.
#
# One or more source files to be compiled into the binary must be declared
# after the SOURCES keyword.
function(add_libfuzzer_test name)
set(multi_arg_options "SOURCES")
cmake_parse_arguments(
"add_libfuzzer_test" "" "" "${multi_arg_options}" ${ARGN})
if ("${add_libfuzzer_test_SOURCES}" STREQUAL "")
message(FATAL_ERROR "Source files must be specified")
endif()
add_executable(LLVMFuzzer-${name}
${add_libfuzzer_test_SOURCES}
)
target_link_libraries(LLVMFuzzer-${name} LLVMFuzzer)
# Place binary where llvm-lit expects to find it
set_target_properties(LLVMFuzzer-${name}
PROPERTIES RUNTIME_OUTPUT_DIRECTORY
"${CMAKE_BINARY_DIR}/lib/Fuzzer/test"
)
add_dependencies(TestBinaries LLVMFuzzer-${name})
endfunction()

###############################################################################
# Basic tests
###############################################################################

set(Tests
AbsNegAndConstantTest
AbsNegAndConstant64Test
AccumulateAllocationsTest
BadStrcmpTest
BogusInitializeTest
BufferOverflowOnInput
CallerCalleeTest
CleanseTest
CounterTest
CustomCrossOverAndMutateTest
CustomCrossOverTest
CustomMutatorTest
CxxStringEqTest
DeepRecursionTest
DivTest
EmptyTest
EquivalenceATest
EquivalenceBTest
FlagsTest
FourIndependentBranchesTest
FullCoverageSetTest
InitializeTest
Memcmp64BytesTest
MemcmpTest
LeakTest
LeakTimeoutTest
LoadTest
NullDerefTest
NullDerefOnEmptyTest
NthRunCrashTest
OneHugeAllocTest
OutOfMemoryTest
OutOfMemorySingleLargeMallocTest
OverwriteInputTest
RepeatedMemcmp
RepeatedBytesTest
SimpleCmpTest
SimpleDictionaryTest
SimpleHashTest
SimpleTest
SimpleThreadedTest
SingleByteInputTest
SingleMemcmpTest
SingleStrcmpTest
SingleStrncmpTest
SpamyTest
ShrinkControlFlowTest
ShrinkControlFlowSimpleTest
ShrinkValueProfileTest
StrcmpTest
StrncmpOOBTest
StrncmpTest
StrstrTest
SwapCmpTest
SwitchTest
Switch2Test
TableLookupTest
ThreadedLeakTest
ThreadedTest
TimeoutTest
TimeoutEmptyTest
TraceMallocTest
TwoDifferentBugsTest
)

if(APPLE OR MSVC)
# LeakSanitizer is not supported on OSX and Windows right now
set(HAS_LSAN 0)
Expand All @@ -148,35 +30,14 @@ else()
set(HAS_LSAN 1)
endif()

foreach(Test ${Tests})
add_libfuzzer_test(${Test} SOURCES ${Test}.cpp)
endforeach()

function(test_export_symbol target symbol)
if(MSVC)
set_target_properties(LLVMFuzzer-${target} PROPERTIES LINK_FLAGS
"-export:${symbol}")
endif()
endfunction()

test_export_symbol(FlagsTest "LLVMFuzzerInitialize")
test_export_symbol(InitializeTest "LLVMFuzzerInitialize")
test_export_symbol(BogusInitializeTest "LLVMFuzzerInitialize")
test_export_symbol(CustomCrossOverTest "LLVMFuzzerCustomCrossOver")
test_export_symbol(CustomMutatorTest "LLVMFuzzerCustomMutator")

###############################################################################
# Unit tests
###############################################################################

add_executable(LLVMFuzzer-Unittest
FuzzerUnittest.cpp
)
add_custom_target(FuzzerUnitTests)
set_target_properties(FuzzerUnitTests PROPERTIES FOLDER "libFuzzer tests")

add_executable(LLVMFuzzer-StandaloneInitializeTest
InitializeTest.cpp
../standalone/StandaloneFuzzTargetMain.c
)
add_executable(LLVMFuzzer-Unittest FuzzerUnittest.cpp)

target_link_libraries(LLVMFuzzer-Unittest
gtest
Expand All @@ -188,64 +49,13 @@ target_include_directories(LLVMFuzzer-Unittest PRIVATE
"${LLVM_MAIN_SRC_DIR}/utils/unittest/googletest/include"
)

add_dependencies(TestBinaries LLVMFuzzer-Unittest)
set_target_properties(LLVMFuzzer-Unittest
PROPERTIES RUNTIME_OUTPUT_DIRECTORY
"${CMAKE_CURRENT_BINARY_DIR}"
)

add_dependencies(TestBinaries LLVMFuzzer-StandaloneInitializeTest)
set_target_properties(LLVMFuzzer-StandaloneInitializeTest
PROPERTIES RUNTIME_OUTPUT_DIRECTORY
"${CMAKE_CURRENT_BINARY_DIR}"
)

###############################################################################
# Additional tests
###############################################################################

include_directories(..)

# add_subdirectory(uninstrumented)
add_subdirectory(no-coverage)
add_subdirectory(trace-pc)
add_subdirectory(ubsan)
if (NOT MSVC)
add_subdirectory(inline-8bit-counters)
endif()

add_library(LLVMFuzzer-DSO1 SHARED DSO1.cpp)
add_library(LLVMFuzzer-DSO2 SHARED DSO2.cpp)

add_executable(LLVMFuzzer-DSOTest
DSOTestMain.cpp
DSOTestExtra.cpp)

target_link_libraries(LLVMFuzzer-DSOTest
LLVMFuzzer-DSO1
LLVMFuzzer-DSO2
LLVMFuzzer
)

set_target_properties(LLVMFuzzer-DSOTest PROPERTIES RUNTIME_OUTPUT_DIRECTORY
"${CMAKE_BINARY_DIR}/lib/Fuzzer/test")

if(MSVC)
set_output_directory(LLVMFuzzer-DSO1
BINARY_DIR "${CMAKE_BINARY_DIR}/lib/Fuzzer/test"
LIBRARY_DIR "${CMAKE_BINARY_DIR}/lib/Fuzzer/test")
set_output_directory(LLVMFuzzer-DSO2
BINARY_DIR "${CMAKE_BINARY_DIR}/lib/Fuzzer/test"
LIBRARY_DIR "${CMAKE_BINARY_DIR}/lib/Fuzzer/test")
else(MSVC)
set_output_directory(LLVMFuzzer-DSO1
LIBRARY_DIR "${CMAKE_BINARY_DIR}/lib/Fuzzer/lib")
set_output_directory(LLVMFuzzer-DSO2
LIBRARY_DIR "${CMAKE_BINARY_DIR}/lib/Fuzzer/lib")
endif()

add_dependencies(TestBinaries LLVMFuzzer-DSOTest)

###############################################################################
# Configure lit to run the tests
#
Expand All @@ -257,23 +67,36 @@ if (MSVC)
set(LIBFUZZER_POSIX 0)
endif()

# Use just-built Clang to compile/link tests on all platforms, except for
# Windows where we need to use clang-cl instead.
if(NOT MSVC)
set(LIBFUZZER_TEST_COMPILER ${LLVM_RUNTIME_OUTPUT_INTDIR}/clang)
set(LIBFUZZER_TEST_CXX_COMPILER ${LLVM_RUNTIME_OUTPUT_INTDIR}/clang++)
else()
set(LIBFUZZER_TEST_COMPILER ${LLVM_RUNTIME_OUTPUT_INTDIR}/clang.exe)
set(LIBFUZZER_TEST_CXX_COMPILER ${LLVM_RUNTIME_OUTPUT_INTDIR}/clang++.exe)
endif()

# LIT-based libFuzzer tests.
configure_lit_site_cfg(
${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in
${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg
)

# libFuzzer unit tests.
configure_lit_site_cfg(
${CMAKE_CURRENT_SOURCE_DIR}/unit/lit.site.cfg.in
${CMAKE_CURRENT_BINARY_DIR}/unit/lit.site.cfg
)

add_lit_testsuite(check-fuzzer "Running Fuzzer tests"
${CMAKE_CURRENT_BINARY_DIR}
DEPENDS TestBinaries
DEPENDS LLVMFuzzer-Unittest
)


# Don't add dependencies on Windows. The linker step would fail on Windows,
# since cmake will use link.exe for linking and won't include compiler-rt libs.
if(NOT MSVC)
add_dependencies(check-fuzzer FileCheck sancov not llvm-symbolizer)
add_dependencies(check-fuzzer LLVMFuzzer asan clang llvm-symbolizer FileCheck sancov not)
endif()
14 changes: 8 additions & 6 deletions llvm/lib/Fuzzer/test/afl-driver-extra-stats.test
View file
Original file line number Diff line number Diff line change
@@ -1,30 +1,32 @@
REQUIRES: posix

RUN: %no_fuzzer_cpp_compiler -fno-sanitize-coverage=edge,trace-cmp,indirect-calls,8bit-counters,trace-pc-guard %S/AFLDriverTest.cpp %S/../afl/afl_driver.cpp -o %t-AFLDriverTest

; Test that not specifying an extra stats file isn't broken.
RUN: unset AFL_DRIVER_EXTRA_STATS_FILENAME
RUN: AFLDriverTest
RUN: %t-AFLDriverTest

; Test that specifying an invalid extra stats file causes a crash.
RUN: ASAN_OPTIONS= AFL_DRIVER_EXTRA_STATS_FILENAME=%T not --crash AFLDriverTest
RUN: ASAN_OPTIONS= AFL_DRIVER_EXTRA_STATS_FILENAME=%T not --crash %t-AFLDriverTest

; Test that specifying a corrupted stats file causes a crash.
echo "peak_rss_mb :0" > %t
ASAN_OPTIONS= AFL_DRIVER_EXTRA_STATS_FILENAME=%t not --crash AFLDriverTest
ASAN_OPTIONS= AFL_DRIVER_EXTRA_STATS_FILENAME=%t not --crash %t-AFLDriverTest

; Test that specifying a valid nonexistent stats file works.
RUN: rm -f %t
RUN: AFL_DRIVER_EXTRA_STATS_FILENAME=%t AFLDriverTest
RUN: AFL_DRIVER_EXTRA_STATS_FILENAME=%t %t-AFLDriverTest
RUN: [[ $(grep "peak_rss_mb\|slowest_unit_time_sec" %t | wc -l) -eq 2 ]]

; Test that specifying a valid preexisting stats file works.
RUN: printf "peak_rss_mb : 0\nslowest_unit_time_sec: 0\n" > %t
RUN: AFL_DRIVER_EXTRA_STATS_FILENAME=%t AFLDriverTest
RUN: AFL_DRIVER_EXTRA_STATS_FILENAME=%t %t-AFLDriverTest
; Check that both lines were printed.
RUN: [[ $(grep "peak_rss_mb\|slowest_unit_time_sec" %t | wc -l) -eq 2 ]]

; Test that peak_rss_mb and slowest_unit_time_in_secs are only updated when necessary.
; Check that both lines have 9999 since there's no way we have exceeded that
; amount of time or virtual memory.
RUN: printf "peak_rss_mb : 9999\nslowest_unit_time_sec: 9999\n" > %t
RUN: AFL_DRIVER_EXTRA_STATS_FILENAME=%t AFLDriverTest
RUN: AFL_DRIVER_EXTRA_STATS_FILENAME=%t %t-AFLDriverTest
RUN: [[ $(grep "9999" %t | wc -l) -eq 2 ]]
8 changes: 5 additions & 3 deletions llvm/lib/Fuzzer/test/afl-driver-stderr.test
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
REQUIRES: posix

RUN: %no_fuzzer_cpp_compiler -fno-sanitize-coverage=edge,trace-cmp,indirect-calls,8bit-counters,trace-pc-guard %S/AFLDriverTest.cpp %S/../afl/afl_driver.cpp -o %t-AFLDriverTest

; Test that not specifying a stderr file isn't broken.
RUN: unset AFL_DRIVER_STDERR_DUPLICATE_FILENAME
RUN: AFLDriverTest
RUN: %t-AFLDriverTest

; Test that specifying an invalid file causes a crash.
RUN: ASAN_OPTIONS= AFL_DRIVER_STDERR_DUPLICATE_FILENAME="%T" not --crash AFLDriverTest
RUN: ASAN_OPTIONS= AFL_DRIVER_STDERR_DUPLICATE_FILENAME="%T" not --crash %t-AFLDriverTest

; Test that a file is created when specified as the duplicate stderr.
RUN: AFL_DRIVER_STDERR_DUPLICATE_FILENAME=%t AFLDriverTest
RUN: AFL_DRIVER_STDERR_DUPLICATE_FILENAME=%t %t-AFLDriverTest
RUN: stat %t
Loading

0 comments on commit 8ecdd7b

Please sign in to comment.