[flang] Regression tests configuration for f18 repository (flang-compiler/f18#861)

The configuration for the tests are in lit.* files.
The lit tests rely on the presence of llvm-lit,FileCheck, not and  count.
When building LLVM add:
-DLLVM_INSTALL_UTILS=On at the cmake command.
LLVM_LIT is found by setting LLVM_EXTERNAL_LIT in f18 CMakeLists.txt.

This patch:
  * Uses LLVM_EXTERNAL_LIT
  * Adds regression tests configurations
  * Adds a proof of concept regression test

The regression test needs to have the Utils build in LLVM.
This is done by adding:
  -DLLVM_INSTALL_UTILS=On
to the LLVM build cmake.

Signed-off-by: Caroline Concatto <caroline.concatto@arm.com>

Original-commit: flang-compiler/f18@a58c6067a1
Reviewed-on: https://github.com/flang-compiler/f18/pull/861

flang/CMakeLists.txt

@@ -50,6 +50,10 @@ message(STATUS "Build Type: ${CMAKE_BUILD_TYPE}" )
 find_package(LLVM REQUIRED CONFIG)
 message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION} in ${LLVM_DIR}")
 
+list(APPEND CMAKE_MODULE_PATH ${LLVM_DIR})
+
+include(AddLLVM)
+
 # Get names for the LLVM libraries
 #
 # The full list of LLVM components can be obtained with
@@ -69,6 +73,9 @@ message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION} in ${LLVM_DIR}")
 include_directories(${LLVM_INCLUDE_DIRS})
 add_definitions(${LLVM_DEFINITIONS})
 
+# LLVM_LIT_EXTERNAL store in cache so it could be used by AddLLVM.cmake
+set(LLVM_EXTERNAL_LIT ${LLVM_TOOLS_BINARY_DIR}/llvm-lit CACHE STRING "Command used to spawn lit")
+
 if(LINK_WITH_FIR)
   message(STATUS "Linking driver with FIR and LLVM")
   llvm_map_components_to_libnames(LLVM_COMMON_LIBS support)
@@ -115,6 +122,7 @@ message(STATUS "FLANG version: ${FLANG_VERSION}")
 
 set(FLANG_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 set(FLANG_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
+set(FLANG_TOOLS_DIR  ${FLANG_BINARY_DIR}/tools/f18/bin)
 
 include_directories(BEFORE
   ${FLANG_BINARY_DIR}/include
@@ -128,6 +136,7 @@ add_subdirectory(lib)
 add_subdirectory(runtime)
 add_subdirectory(test)
 add_subdirectory(tools)
+add_subdirectory(test-lit)
 
 configure_file(
   ${FLANG_SOURCE_DIR}/include/flang/Config/config.h.cmake

flang/README.md

@@ -83,6 +83,16 @@ where `LLVM_INSTALLATION_DIR` is
 the top-level directory
 where llvm is installed.
 
+### LLVM dependency for lit Regression tests
+
+F18 has tests that use the lit framework, these tests rely on the
+presence of llvm tools as llvm-lit, FileCheck, and others.
+These tools are installed when LLVM build set:
+```
+LLVM_INSTALL_UTILS=On
+```
+to run the regression tests on f18.
+
 ### Building f18 with GCC
 
 By default,
@@ -169,3 +179,22 @@ cd ~/f18/build
 cmake -DLLVM_DIR=$LLVM ~/f18/src
 make
 ```
+
+### How to Run the Regression Tests
+
+To run all tests:
+```
+cd ~/f18/build
+cmake -DLLVM_DIR=$LLVM ~/f18/src
+make check-all
+```
+
+To run individual regression tests llvm-lit needs to know the lit
+configuration for f18. The parameters in charge of this are:
+flang_site_config and flang_config. And they can be set as shown bellow:
+```
+<path-to-llvm-lit>/llvm-lit \
+ --param flang_site_config=<path-to-f18-build>/test-lit/lit.site.cfg.py \
+ --param flang_config=<path-to-f18-build>/test-lit/lit.cfg.py \
+  <path-to-fortran-test>
+```

flang/test-lit/CMakeLists.txt

@@ -0,0 +1,28 @@
+# Test runner infrastructure for Flang. This configures the Flang test trees
+# for use by Lit, and delegates to LLVM's lit test handlers.
+
+configure_lit_site_cfg(
+  ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
+  ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
+  MAIN_CONFIG
+  ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
+)
+
+set(FLANG_TEST_PARAMS
+  flang_site_config=${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py)
+
+set(FLANG_TEST_DEPENDS
+  flang
+  f18
+  llvm-lit
+  FileCheck
+  count
+  not
+)
+add_lit_testsuite(check-all "Running the Flang regression tests"
+  ${CMAKE_CURRENT_BINARY_DIR}
+  PARAMS ${FLANG_TEST_PARAMS}
+  DEPENDS ${FLANG_TEST_DEPENS}
+  )
+set_target_properties(check-all PROPERTIES FOLDER "Tests")
+

flang/test-lit/driver/version_test.f90

@@ -0,0 +1,7 @@
+! Check that lit configuration works by checking the compiler version
+
+! RUN: %f18 -V 2>&1 | FileCheck  -check-prefix=VERSION %s
+! VERSION-NOT:{{![[:space:]]}}
+! VERSION:{{[[:space:]]}}
+! VERSION-SAME:f18 compiler (under development)
+! VERSION-EMPTY:

flang/test-lit/lit.cfg.py

@@ -0,0 +1,69 @@
+# -*- Python -*-
+
+import os
+import platform
+import re
+import subprocess
+import sys
+
+import lit.formats
+import lit.util
+
+from lit.llvm import llvm_config
+from lit.llvm.subst import ToolSubst
+from lit.llvm.subst import FindTool
+
+# Configuration file for the 'lit' test runner.
+
+# name: The name of this test suite.
+config.name = 'Flang'
+
+# testFormat: The test format to use to interpret tests.
+#
+# For now we require '&&' between commands, until they get globally killed and
+# the test runner updated.
+config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
+
+
+# suffixes: A list of file extensions to treat as test files.
+config.suffixes = ['.f', '.F', '.ff','.FOR', '.for', '.f77', '.f90', '.F90',
+                   '.ff90', '.f95', '.F95', '.ff95', '.fpp', '.FPP', '.cuf',
+                   '.CUF', '.f18', '.F18']
+
+# test_source_root: The root path where tests are located.
+config.test_source_root = os.path.dirname(__file__)
+
+
+# test_exec_root: The root path where tests should be run.
+config.test_exec_root = os.path.join(config.flang_obj_root, 'test-lit')
+
+config.substitutions.append(('%PATH%', config.environment['PATH']))
+
+llvm_config.use_default_substitutions()
+
+# excludes: A list of directories to exclude from the testsuite. The 'Inputs'
+# subdirectories contain auxiliary inputs for various tests in their parent
+# directories.
+config.excludes = ['Inputs', 'CMakeLists.txt', 'README.txt', 'LICENSE.txt']
+
+# test_source_root: The root path where tests are located.
+config.test_source_root = os.path.dirname(__file__)
+
+# test_exec_root: The root path where tests should be run.
+config.test_exec_root = os.path.join(config.flang_obj_root, 'test-lit')
+
+# Tweak the PATH to include the tools dir.
+llvm_config.with_environment('PATH', config.flang_tools_dir, append_path=True)
+llvm_config.with_environment('PATH', config.llvm_tools_dir, append_path=True)
+
+# For each occurrence of a flang tool name, replace it with the full path to
+# the build directory holding that tool.  We explicitly specify the directories
+# to search to ensure that we get the tools just built and not some random
+# tools that might happen to be in the user's PATH.
+tool_dirs = [config.llvm_tools_dir, config.flang_tools_dir]
+
+tools = [ToolSubst('%flang', command=FindTool('flang'), unresolved='fatal'),
+         ToolSubst('%f18', command=FindTool('f18'), unresolved='fatal')]
+
+llvm_config.add_tool_substitutions(tools, tool_dirs)
+

flang/test-lit/lit.site.cfg.py.in

@@ -0,0 +1,25 @@
+@LIT_SITE_CFG_IN_HEADER@
+
+import sys
+
+config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
+config.flang_obj_root = "@FLANG_BINARY_DIR@"
+config.flang_src_dir = "@FLANG_SOURCE_DIR@"
+config.flang_tools_dir = "@FLANG_TOOLS_DIR@"
+config.python_executable = "@PYTHON_EXECUTABLE@"
+
+# Support substitution of the tools_dir with user parameters. This is
+# used when we can't determine the tool dir at configuration time.
+try:
+    config.llvm_tools_dir = config.llvm_tools_dir % lit_config.params
+    config.flang_tools_dir = config.flang_tools_dir % lit_config.params
+except KeyError:
+    e = sys.exc_info()[1]
+    key, = e.args
+    lit_config.fatal("unable to find %r parameter, use '--param=%s=VALUE'" % (key,key))
+
+import lit.llvm
+lit.llvm.initialize(lit_config, config)
+
+# Let the main config do the real work.
+lit_config.load_config(config, "@FLANG_SOURCE_DIR@/test-lit/lit.cfg.py")