[LIT] Have lit run the lldb test suite

This is the first in what will hopefully become a series of patches to replace the driver logic in dotest.py with LIT. The motivation for this change is that there's no point in maintaining two driver implementations. Since all of the LLVM projects are using lit, this is the obvious choice. Obviously the goal is maintain full compatibility with the functionality offered by dotest. As such we won't be removing anything until that point has been reached. This patch is the initial attempt (referred to as v1) to run the lldb test suite with lit. To do so we introduced a custom LLDB test format that invokes dotest.py with a single test file. Differential revision: https://reviews.llvm.org/D45333 llvm-svn: 330275
llvm · Apr 18, 2018 · 1a928f3 · 1a928f3
1 parent 75a4e52
commit 1a928f3
Show file tree

Hide file tree

Showing 4 changed files with 133 additions and 5 deletions.
diff --git a/lldb/lit/Suite/lit.cfg b/lldb/lit/Suite/lit.cfg
@@ -0,0 +1,29 @@
+# -*- Python -*-
+
+# Configuration file for the 'lit' test runner.
+
+import os
+
+import lit.formats
+
+# name: The name of this test suite.
+config.name = 'lldb-Suite'
+
+# suffixes: A list of file extensions to treat as test files.
+config.suffixes = ['.py']
+
+# test_source_root: The root path where tests are located.
+# test_exec_root: The root path where tests should be run.
+config.test_source_root = os.path.join(config.lldb_src_root, 'packages','Python','lldbsuite','test')
+config.test_exec_root = config.test_source_root
+
+# Build dotest command.
+dotest_cmd = [config.dotest_path, '-q']
+dotest_cmd.extend(config.dotest_args_str.split(';'))
+
+# Load LLDB test format.
+sys.path.append(os.path.join(config.lldb_src_root, "lit", "Suite"))
+import lldbtest
+
+# testFormat: The test format to use to interpret tests.
+config.test_format = lldbtest.LLDBTest(dotest_cmd)
diff --git a/lldb/lit/Suite/lit.site.cfg.in b/lldb/lit/Suite/lit.site.cfg.in
@@ -0,0 +1,28 @@
+@LIT_SITE_CFG_IN_HEADER@
+
+config.test_exec_root = "@LLVM_BINARY_DIR@"
+config.llvm_src_root = "@LLVM_SOURCE_DIR@"
+config.llvm_obj_root = "@LLVM_BINARY_DIR@"
+config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
+config.llvm_libs_dir = "@LLVM_LIBS_DIR@"
+config.llvm_build_mode = "@LLVM_BUILD_MODE@"
+config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
+config.lldb_obj_root = "@LLDB_BINARY_DIR@"
+config.lldb_src_root = "@LLDB_SOURCE_DIR@"
+config.target_triple = "@TARGET_TRIPLE@"
+config.python_executable = "@PYTHON_EXECUTABLE@"
+config.dotest_path = "@LLDB_SOURCE_DIR@/test/dotest.py"
+config.dotest_args_str = "@LLDB_DOTEST_ARGS@"
+
+# Support substitution of the tools and libs dirs with user parameters. This is
+# used when we can't determine the tool dir at configuration time.
+try:
+    config.llvm_tools_dir = config.llvm_tools_dir % lit_config.params
+    config.llvm_libs_dir = config.llvm_libs_dir % lit_config.params
+    config.llvm_build_mode = config.llvm_build_mode % lit_config.params
+except KeyError as e:
+    key, = e.args
+    lit_config.fatal("unable to find %r parameter, use '--param=%s=VALUE'" % (key,key))
+
+# Let the main config do the real work.
+lit_config.load_config(config, "@LLDB_SOURCE_DIR@/lit/Suite/lit.cfg")
diff --git a/lldb/lit/Suite/lldbtest.py b/lldb/lit/Suite/lldbtest.py
@@ -0,0 +1,64 @@
+from __future__ import absolute_import
+import os
+
+import subprocess
+import sys
+
+import lit.Test
+import lit.TestRunner
+import lit.util
+from lit.formats.base import TestFormat
+
+
+class LLDBTest(TestFormat):
+    def __init__(self, dotest_cmd):
+        self.dotest_cmd = dotest_cmd
+
+    def getTestsInDirectory(self, testSuite, path_in_suite, litConfig,
+                            localConfig):
+        source_path = testSuite.getSourcePath(path_in_suite)
+        for filename in os.listdir(source_path):
+            # Ignore dot files and excluded tests.
+            if (filename.startswith('.') or filename in localConfig.excludes):
+                continue
+
+            # Ignore files that don't start with 'Test'.
+            if not filename.startswith('Test'):
+                continue
+
+            filepath = os.path.join(source_path, filename)
+            if not os.path.isdir(filepath):
+                base, ext = os.path.splitext(filename)
+                if ext in localConfig.suffixes:
+                    yield lit.Test.Test(testSuite, path_in_suite +
+                                        (filename, ), localConfig)
+
+    def execute(self, test, litConfig):
+        if litConfig.noExecute:
+            return lit.Test.PASS, ''
+
+        if test.config.unsupported:
+            return (lit.Test.UNSUPPORTED, 'Test is unsupported')
+
+        testPath, testFile = os.path.split(test.getSourcePath())
+        cmd = self.dotest_cmd + [testPath, '-p', testFile]
+
+        try:
+            out, err, exitCode = lit.util.executeCommand(
+                cmd,
+                env=test.config.environment,
+                timeout=litConfig.maxIndividualTestTime)
+        except lit.util.ExecuteCommandTimeoutException:
+            return (lit.Test.TIMEOUT, 'Reached timeout of {} seconds'.format(
+                litConfig.maxIndividualTestTime))
+
+        if exitCode:
+            return lit.Test.FAIL, out + err
+
+        passing_test_line = 'RESULT: PASSED'
+        if passing_test_line not in out and passing_test_line not in err:
+            msg = ('Unable to find %r in dotest output:\n\n%s%s' %
+                   (passing_test_line, out, err))
+            return lit.Test.UNRESOLVED, msg
+
+        return lit.Test.PASS, ''
diff --git a/lldb/test/CMakeLists.txt b/lldb/test/CMakeLists.txt
@@ -130,11 +130,7 @@ add_python_test_target(check-lldb-single
 
 # If tests crash cause LLDB to crash, or things are otherwise unstable, or if machine-parsable
 # output is desired (i.e. in continuous integration contexts) check-lldb-single is a better target.
-add_python_test_target(check-lldb
-  ${LLDB_SOURCE_DIR}/test/dotest.py
-  "-q;${LLDB_DOTEST_ARGS}"
-  "Testing LLDB (parallel execution, with a separate subprocess per test)"
-  )
+add_custom_target(check-lldb)
 
 # Generate a wrapper for dotest.py in the bin directory.
 # We need configure_file to substitute variables.
@@ -153,6 +149,17 @@ file(GENERATE
 add_custom_target(lldb-dotest)
 add_dependencies(lldb-dotest ${LLDB_TEST_DEPS})
 
+configure_file(
+  ${CMAKE_CURRENT_SOURCE_DIR}/../lit/Suite/lit.site.cfg.in
+  ${CMAKE_CURRENT_BINARY_DIR}/../lit/Suite/lit.site.cfg
+  )
+file(GENERATE
+  OUTPUT
+  ${CMAKE_CURRENT_BINARY_DIR}/../lit/Suite/lit.site.cfg
+  INPUT
+  ${CMAKE_CURRENT_BINARY_DIR}/../lit/Suite/lit.site.cfg
+  )
+
 # If we're building with an in-tree clang, then list clang as a dependency
 # to run tests.
 if (TARGET clang)