[AST] Add generator for source location introspection

Generate a json file containing descriptions of AST classes and their
public accessors which return SourceLocation or SourceRange.

Use the JSON file to generate a C++ API and implementation for accessing
the source locations and method names for accessing them for a given AST
node.

This new API can be used to implement 'srcloc' output in clang-query:

  http://ce.steveire.com/z/m_kTIo

The JSON file can also be used to generate bindings for other languages,
such as Python and Javascript:

  https://steveire.wordpress.com/2019/04/30/the-future-of-ast-matching

In this first version of this feature, only the accessors for Stmt
classes are generated, not Decls, TypeLocs etc.  Those can be added
after this change is reviewed, as this change is mostly about
infrastructure of these code generators.

Also in this version, the platforms/cmake configurations are excluded as
much as possible so that support can be added iteratively.  Currently a
break on any platform causes a revert of the entire feature.  This way,
the `OR WIN32` can be removed in a future commit and if it breaks the
buildbots, only that commit gets reverted, making the entire process
easier to manage.

Differential Revision: https://reviews.llvm.org/D93164
This commit is contained in:
Stephen Kelly 2021-03-14 16:18:54 +00:00
parent 463863fffe
commit 477e4b9746
16 changed files with 1032 additions and 0 deletions

View File

@ -0,0 +1,85 @@
//===- NodeIntrospection.h ------------------------------------*- C++ -*---===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file contains the implementation of the NodeIntrospection.
//
//===----------------------------------------------------------------------===//
#ifndef LLVM_CLANG_TOOLING_NODEINTROSPECTION_H
#define LLVM_CLANG_TOOLING_NODEINTROSPECTION_H
#include "clang/AST/ASTTypeTraits.h"
#include "clang/AST/DeclarationName.h"
#include <memory>
#include <set>
namespace clang {
class Stmt;
namespace tooling {
class LocationCall {
public:
enum LocationCallFlags { NoFlags, ReturnsPointer, IsCast };
LocationCall(std::shared_ptr<LocationCall> on, std::string name,
LocationCallFlags flags = NoFlags)
: m_on(on), m_name(name), m_flags(flags) {}
LocationCall(std::shared_ptr<LocationCall> on, std::string name,
std::vector<std::string> const &args,
LocationCallFlags flags = NoFlags)
: m_on(on), m_name(name), m_flags(flags) {}
LocationCall *on() const { return m_on.get(); }
StringRef name() const { return m_name; }
std::vector<std::string> const &args() const { return m_args; }
bool returnsPointer() const { return m_flags & ReturnsPointer; }
bool isCast() const { return m_flags & IsCast; }
private:
std::shared_ptr<LocationCall> m_on;
std::string m_name;
std::vector<std::string> m_args;
LocationCallFlags m_flags;
};
class LocationCallFormatterCpp {
public:
static std::string format(LocationCall *Call);
};
namespace internal {
struct RangeLessThan {
bool operator()(
std::pair<SourceRange, std::shared_ptr<LocationCall>> const &LHS,
std::pair<SourceRange, std::shared_ptr<LocationCall>> const &RHS) const;
};
} // namespace internal
template <typename T, typename U, typename Comp = std::less<std::pair<T, U>>>
using UniqueMultiMap = std::set<std::pair<T, U>, Comp>;
using SourceLocationMap =
UniqueMultiMap<SourceLocation, std::shared_ptr<LocationCall>>;
using SourceRangeMap =
UniqueMultiMap<SourceRange, std::shared_ptr<LocationCall>,
internal::RangeLessThan>;
struct NodeLocationAccessors {
SourceLocationMap LocationAccessors;
SourceRangeMap RangeAccessors;
};
namespace NodeIntrospection {
NodeLocationAccessors GetLocations(clang::Stmt const *Object);
NodeLocationAccessors GetLocations(clang::DynTypedNode const &Node);
} // namespace NodeIntrospection
} // namespace tooling
} // namespace clang
#endif

View File

@ -8,10 +8,107 @@ add_subdirectory(Core)
add_subdirectory(Inclusions)
add_subdirectory(Refactoring)
add_subdirectory(ASTDiff)
add_subdirectory(DumpTool)
add_subdirectory(Syntax)
add_subdirectory(DependencyScanning)
add_subdirectory(Transformer)
find_package(Python3 COMPONENTS Interpreter)
# Replace the last lib component of the current binary directory with include
string(FIND ${CMAKE_CURRENT_BINARY_DIR} "/lib/" PATH_LIB_START REVERSE)
if(PATH_LIB_START EQUAL -1)
message(FATAL_ERROR "Couldn't find lib component in binary directory")
endif()
math(EXPR PATH_LIB_END "${PATH_LIB_START}+5")
string(SUBSTRING ${CMAKE_CURRENT_BINARY_DIR} 0 ${PATH_LIB_START} PATH_HEAD)
string(SUBSTRING ${CMAKE_CURRENT_BINARY_DIR} ${PATH_LIB_END} -1 PATH_TAIL)
string(CONCAT BINARY_INCLUDE_DIR ${PATH_HEAD} "/include/clang/" ${PATH_TAIL})
if (NOT Python3_EXECUTABLE
OR WIN32
OR APPLE
OR GENERATOR_IS_MULTI_CONFIG
OR NOT LLVM_NATIVE_ARCH IN_LIST LLVM_TARGETS_TO_BUILD
)
file(GENERATE OUTPUT ${BINARY_INCLUDE_DIR}/NodeIntrospection.inc
CONTENT "
namespace clang {
namespace tooling {
NodeLocationAccessors NodeIntrospection::GetLocations(clang::Stmt const *) {
return {};
}
NodeLocationAccessors
NodeIntrospection::GetLocations(clang::DynTypedNode const &) {
return {};
}
} // namespace tooling
} // namespace clang
"
)
else()
# The generation of ASTNodeAPI.json takes a long time in a
# Debug build due to parsing AST.h. Disable the processing
# but setting CLANG_TOOLING_BUILD_AST_INTROSPECTION as an
# internal hidden setting to override.
# When the processing is disabled, a trivial/empty JSON
# file is generated by clang-ast-dump and generate_cxx_src_locs.py
# generates the same API, but with a trivial implementation.
option(CLANG_TOOLING_BUILD_AST_INTROSPECTION "Enable AST introspection" TRUE)
set(skip_expensive_processing $<OR:$<CONFIG:Debug>,$<NOT:$<BOOL:${CLANG_TOOLING_BUILD_AST_INTROSPECTION}>>>)
file(GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ASTTU.cpp
CONTENT "
#include <clang/AST/AST.h>
")
add_custom_command(
COMMENT Generate ASTNodeAPI.json
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ASTNodeAPI.json
DEPENDS clang-ast-dump clang-resource-headers
COMMAND
$<TARGET_FILE:clang-ast-dump>
# Skip this in debug mode because parsing AST.h is too slow
--skip-processing=${skip_expensive_processing}
--astheader=${CMAKE_CURRENT_BINARY_DIR}/ASTTU.cpp
-I ${CMAKE_BINARY_DIR}/lib/clang/${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}.${LLVM_VERSION_PATCH}/include
-I ${CMAKE_SOURCE_DIR}/../clang/include
-I ${CMAKE_BINARY_DIR}/tools/clang/include/
-I ${CMAKE_BINARY_DIR}/include
-I ${CMAKE_SOURCE_DIR}/include
--json-output-path ${CMAKE_CURRENT_BINARY_DIR}/ASTNodeAPI.json
)
add_custom_target(run-ast-api-dump-tool
DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/ASTNodeAPI.json
)
add_custom_command(
COMMENT Generate NodeIntrospection.inc
OUTPUT ${BINARY_INCLUDE_DIR}/NodeIntrospection.inc
DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/ASTNodeAPI.json ${CMAKE_CURRENT_SOURCE_DIR}/DumpTool/generate_cxx_src_locs.py
COMMAND
${CMAKE_COMMAND} -E make_directory
${CMAKE_CURRENT_BINARY_DIR}/generated/
COMMAND
${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/DumpTool/generate_cxx_src_locs.py
--json-input-path ${CMAKE_CURRENT_BINARY_DIR}/ASTNodeAPI.json
--output-file NodeIntrospection.inc
--empty-implementation ${skip_expensive_processing}
COMMAND
${CMAKE_COMMAND} -E copy_if_different
${CMAKE_CURRENT_BINARY_DIR}/NodeIntrospection.inc
${BINARY_INCLUDE_DIR}/NodeIntrospection.inc
)
add_custom_target(run-ast-api-generate-tool
DEPENDS
${BINARY_INCLUDE_DIR}/NodeIntrospection.inc
)
endif()
add_clang_library(clangTooling
AllTUsExecution.cpp
ArgumentsAdjusters.cpp
@ -27,6 +124,8 @@ add_clang_library(clangTooling
Refactoring.cpp
RefactoringCallbacks.cpp
StandaloneExecution.cpp
NodeIntrospection.cpp
${BINARY_INCLUDE_DIR}/NodeIntrospection.inc
Tooling.cpp
DEPENDS

View File

@ -0,0 +1,32 @@
//===- APIData.h ---------------------------------------------*- C++ -*----===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef LLVM_CLANG_LIB_TOOLING_DUMPTOOL_APIDATA_H
#define LLVM_CLANG_LIB_TOOLING_DUMPTOOL_APIDATA_H
#include <string>
#include <vector>
namespace clang {
namespace tooling {
struct ClassData {
bool isEmpty() const {
return ASTClassLocations.empty() && ASTClassRanges.empty();
}
std::vector<std::string> ASTClassLocations;
std::vector<std::string> ASTClassRanges;
// TODO: Extend this with locations available via typelocs etc.
};
} // namespace tooling
} // namespace clang
#endif

View File

@ -0,0 +1,170 @@
//===- ASTSrcLocProcessor.cpp --------------------------------*- C++ -*----===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "ASTSrcLocProcessor.h"
#include "clang/Frontend/CompilerInstance.h"
#include "llvm/Support/JSON.h"
using namespace clang::tooling;
using namespace llvm;
using namespace clang::ast_matchers;
ASTSrcLocProcessor::ASTSrcLocProcessor(StringRef JsonPath)
: JsonPath(JsonPath) {
MatchFinder::MatchFinderOptions FinderOptions;
Finder = std::make_unique<MatchFinder>(std::move(FinderOptions));
Finder->addMatcher(
cxxRecordDecl(
isDefinition(),
isSameOrDerivedFrom(
// TODO: Extend this with other clades
namedDecl(hasName("clang::Stmt")).bind("nodeClade")),
optionally(isDerivedFrom(cxxRecordDecl().bind("derivedFrom"))))
.bind("className"),
this);
}
std::unique_ptr<clang::ASTConsumer>
ASTSrcLocProcessor::createASTConsumer(clang::CompilerInstance &Compiler,
StringRef File) {
return Finder->newASTConsumer();
}
llvm::json::Object toJSON(llvm::StringMap<std::vector<StringRef>> const &Obj) {
using llvm::json::toJSON;
llvm::json::Object JsonObj;
for (const auto &Item : Obj) {
JsonObj[Item.first()] = Item.second;
}
return JsonObj;
}
llvm::json::Object toJSON(llvm::StringMap<StringRef> const &Obj) {
using llvm::json::toJSON;
llvm::json::Object JsonObj;
for (const auto &Item : Obj) {
JsonObj[Item.first()] = Item.second;
}
return JsonObj;
}
llvm::json::Object toJSON(ClassData const &Obj) {
llvm::json::Object JsonObj;
if (!Obj.ASTClassLocations.empty())
JsonObj["sourceLocations"] = Obj.ASTClassLocations;
if (!Obj.ASTClassRanges.empty())
JsonObj["sourceRanges"] = Obj.ASTClassRanges;
return JsonObj;
}
llvm::json::Object toJSON(llvm::StringMap<ClassData> const &Obj) {
using llvm::json::toJSON;
llvm::json::Object JsonObj;
for (const auto &Item : Obj) {
if (!Item.second.isEmpty())
JsonObj[Item.first()] = ::toJSON(Item.second);
}
return JsonObj;
}
void WriteJSON(std::string JsonPath,
llvm::StringMap<StringRef> const &ClassInheritance,
llvm::StringMap<std::vector<StringRef>> const &ClassesInClade,
llvm::StringMap<ClassData> const &ClassEntries) {
llvm::json::Object JsonObj;
using llvm::json::toJSON;
JsonObj["classInheritance"] = ::toJSON(ClassInheritance);
JsonObj["classesInClade"] = ::toJSON(ClassesInClade);
JsonObj["classEntries"] = ::toJSON(ClassEntries);
std::error_code EC;
llvm::raw_fd_ostream JsonOut(JsonPath, EC, llvm::sys::fs::F_Text);
if (EC)
return;
llvm::json::Value JsonVal(std::move(JsonObj));
JsonOut << formatv("{0:2}", JsonVal);
}
void ASTSrcLocProcessor::generate() {
WriteJSON(JsonPath, ClassInheritance, ClassesInClade, ClassEntries);
}
std::vector<std::string>
CaptureMethods(std::string TypeString, const clang::CXXRecordDecl *ASTClass,
const MatchFinder::MatchResult &Result) {
auto publicAccessor = [](auto... InnerMatcher) {
return cxxMethodDecl(isPublic(), parameterCountIs(0), isConst(),
InnerMatcher...);
};
auto BoundNodesVec =
match(findAll(publicAccessor(ofClass(equalsNode(ASTClass)),
returns(asString(TypeString)))
.bind("classMethod")),
*ASTClass, *Result.Context);
std::vector<std::string> Methods;
for (const auto &BN : BoundNodesVec) {
if (const auto *Node = BN.getNodeAs<clang::NamedDecl>("classMethod")) {
// Only record the getBeginLoc etc on Stmt etc, because it will call
// more-derived implementations pseudo-virtually.
if ((ASTClass->getName() != "Stmt" && ASTClass->getName() != "Decl") &&
(Node->getName() == "getBeginLoc" || Node->getName() == "getEndLoc" ||
Node->getName() == "getSourceRange")) {
continue;
}
// Only record the getExprLoc on Expr, because it will call
// more-derived implementations pseudo-virtually.
if (ASTClass->getName() != "Expr" && Node->getName() == "getExprLoc") {
continue;
}
Methods.push_back(Node->getName().str());
}
}
return Methods;
}
void ASTSrcLocProcessor::run(const MatchFinder::MatchResult &Result) {
if (const auto *ASTClass =
Result.Nodes.getNodeAs<clang::CXXRecordDecl>("className")) {
StringRef ClassName = ASTClass->getName();
ClassData CD;
const auto *NodeClade =
Result.Nodes.getNodeAs<clang::CXXRecordDecl>("nodeClade");
StringRef CladeName = NodeClade->getName();
if (const auto *DerivedFrom =
Result.Nodes.getNodeAs<clang::CXXRecordDecl>("derivedFrom"))
ClassInheritance[ClassName] = DerivedFrom->getName();
CD.ASTClassLocations =
CaptureMethods("class clang::SourceLocation", ASTClass, Result);
CD.ASTClassRanges =
CaptureMethods("class clang::SourceRange", ASTClass, Result);
if (!CD.isEmpty()) {
ClassEntries[ClassName] = CD;
ClassesInClade[CladeName].push_back(ClassName);
}
}
}

View File

@ -0,0 +1,48 @@
//===- ASTSrcLocProcessor.h ---------------------------------*- C++ -*-----===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef LLVM_CLANG_TOOLING_DUMPTOOL_ASTSRCLOCPROCESSOR_H
#define LLVM_CLANG_TOOLING_DUMPTOOL_ASTSRCLOCPROCESSOR_H
#include "APIData.h"
#include "clang/ASTMatchers/ASTMatchFinder.h"
#include "llvm/ADT/StringRef.h"
#include <memory>
#include <string>
#include <vector>
namespace clang {
class CompilerInstance;
namespace tooling {
class ASTSrcLocProcessor : public ast_matchers::MatchFinder::MatchCallback {
public:
explicit ASTSrcLocProcessor(StringRef JsonPath);
std::unique_ptr<ASTConsumer> createASTConsumer(CompilerInstance &Compiler,
StringRef File);
void generate();
private:
void run(const ast_matchers::MatchFinder::MatchResult &Result) override;
llvm::StringMap<StringRef> ClassInheritance;
llvm::StringMap<std::vector<StringRef>> ClassesInClade;
llvm::StringMap<ClassData> ClassEntries;
std::string JsonPath;
std::unique_ptr<clang::ast_matchers::MatchFinder> Finder;
};
} // namespace tooling
} // namespace clang
#endif

View File

@ -0,0 +1,16 @@
add_clang_executable(clang-ast-dump
ASTSrcLocProcessor.cpp
ClangSrcLocDump.cpp
)
target_link_libraries(clang-ast-dump
PRIVATE
clangAST
clangASTMatchers
clangBasic
clangDriver
clangFrontend
clangSerialization
clangToolingCore
)

View File

@ -0,0 +1,139 @@
//===- ClangSrcLocDump.cpp ------------------------------------*- C++ -*---===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "clang/Basic/Diagnostic.h"
#include "clang/Driver/Compilation.h"
#include "clang/Driver/Driver.h"
#include "clang/Driver/Job.h"
#include "clang/Driver/Options.h"
#include "clang/Driver/Tool.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "clang/Tooling/Tooling.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Host.h"
#include "llvm/Support/JSON.h"
#include "ASTSrcLocProcessor.h"
using namespace clang::tooling;
using namespace clang;
using namespace llvm;
static cl::list<std::string> IncludeDirectories(
"I", cl::desc("Include directories to use while compiling"),
cl::value_desc("directory"), cl::Required, cl::OneOrMore, cl::Prefix);
static cl::opt<std::string>
AstHeaderFile("astheader", cl::desc("AST header to parse API from"),
cl::Required, cl::value_desc("AST header file"));
static cl::opt<bool>
SkipProcessing("skip-processing",
cl::desc("Avoid processing the AST header file"),
cl::Required, cl::value_desc("bool"));
static cl::opt<std::string> JsonOutputPath("json-output-path",
cl::desc("json output path"),
cl::Required,
cl::value_desc("path"));
class ASTSrcLocGenerationAction : public clang::ASTFrontendAction {
public:
ASTSrcLocGenerationAction() : Processor(JsonOutputPath) {}
~ASTSrcLocGenerationAction() { Processor.generate(); }
std::unique_ptr<clang::ASTConsumer>
CreateASTConsumer(clang::CompilerInstance &Compiler,
llvm::StringRef File) override {
return Processor.createASTConsumer(Compiler, File);
}
private:
ASTSrcLocProcessor Processor;
};
int main(int argc, const char **argv) {
cl::ParseCommandLineOptions(argc, argv);
if (SkipProcessing) {
std::error_code EC;
llvm::raw_fd_ostream JsonOut(JsonOutputPath, EC, llvm::sys::fs::F_Text);
if (EC)
return 1;
JsonOut << formatv("{0:2}", llvm::json::Value(llvm::json::Object()));
return 0;
}
std::vector<std::string> Args;
Args.push_back("-cc1");
llvm::transform(IncludeDirectories, std::back_inserter(Args),
[](const std::string &IncDir) { return "-I" + IncDir; });
Args.push_back("-fsyntax-only");
Args.push_back(AstHeaderFile);
std::vector<const char *> Argv(Args.size(), nullptr);
llvm::transform(Args, Argv.begin(),
[](const std::string &Arg) { return Arg.c_str(); });
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
unsigned MissingArgIndex, MissingArgCount;
auto Opts = driver::getDriverOptTable();
auto ParsedArgs = Opts.ParseArgs(llvm::makeArrayRef(Argv).slice(1),
MissingArgIndex, MissingArgCount);
ParseDiagnosticArgs(*DiagOpts, ParsedArgs);
TextDiagnosticPrinter DiagnosticPrinter(llvm::errs(), &*DiagOpts);
DiagnosticsEngine Diagnostics(
IntrusiveRefCntPtr<DiagnosticIDs>(new DiagnosticIDs()), &*DiagOpts,
&DiagnosticPrinter, false);
FileManager Files(FileSystemOptions(), vfs::getRealFileSystem());
auto Driver = std::make_unique<driver::Driver>(
"clang", llvm::sys::getDefaultTargetTriple(), Diagnostics,
"ast-api-dump-tool", &Files.getVirtualFileSystem());
auto Comp = Driver->BuildCompilation(llvm::makeArrayRef(Argv));
if (!Comp)
return 1;
const auto &Jobs = Comp->getJobs();
if (Jobs.size() != 1 || !isa<driver::Command>(*Jobs.begin())) {
SmallString<256> error_msg;
llvm::raw_svector_ostream error_stream(error_msg);
Jobs.Print(error_stream, "; ", true);
return 1;
}
const auto &Cmd = cast<driver::Command>(*Jobs.begin());
const llvm::opt::ArgStringList &CC1Args = Cmd.getArguments();
auto Invocation = std::make_unique<CompilerInvocation>();
CompilerInvocation::CreateFromArgs(*Invocation, CC1Args, Diagnostics);
CompilerInstance Compiler(std::make_shared<clang::PCHContainerOperations>());
Compiler.setInvocation(std::move(Invocation));
Compiler.createDiagnostics(&DiagnosticPrinter, false);
if (!Compiler.hasDiagnostics())
return 1;
Compiler.createSourceManager(Files);
ASTSrcLocGenerationAction ScopedToolAction;
Compiler.ExecuteAction(ScopedToolAction);
Files.clearStatCache();
return 0;
}

View File

@ -0,0 +1,201 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
import sys
import json
import argparse
class Generator(object):
implementationContent = ''
def GeneratePrologue(self):
self.implementationContent += \
"""
/*===- Generated file -------------------------------------------*- C++ -*-===*\
|* *|
|* Introspection of available AST node SourceLocations *|
|* *|
|* Automatically generated file, do not edit! *|
|* *|
\*===----------------------------------------------------------------------===*/
namespace clang {
namespace tooling {
using LocationAndString = SourceLocationMap::value_type;
using RangeAndString = SourceRangeMap::value_type;
"""
def GenerateBaseGetLocationsDeclaration(self, CladeName):
self.implementationContent += \
"""
void GetLocationsImpl(std::shared_ptr<LocationCall> const& Prefix,
clang::{0} const *Object, SourceLocationMap &Locs,
SourceRangeMap &Rngs);
""".format(CladeName)
def GenerateSrcLocMethod(self, ClassName, ClassData):
self.implementationContent += \
"""
static void GetLocations{0}(std::shared_ptr<LocationCall> const& Prefix,
clang::{0} const &Object,
SourceLocationMap &Locs, SourceRangeMap &Rngs)
{{
""".format(ClassName)
if 'sourceLocations' in ClassData:
for locName in ClassData['sourceLocations']:
self.implementationContent += \
"""
Locs.insert(LocationAndString(Object.{0}(),
std::make_shared<LocationCall>(Prefix, "{0}")));
""".format(locName)
self.implementationContent += '\n'
if 'sourceRanges' in ClassData:
for rngName in ClassData['sourceRanges']:
self.implementationContent += \
"""
Rngs.insert(RangeAndString(Object.{0}(),
std::make_shared<LocationCall>(Prefix, "{0}")));
""".format(rngName)
self.implementationContent += '\n'
self.implementationContent += '}\n'
def GenerateFiles(self, OutputFile):
with open(os.path.join(os.getcwd(),
OutputFile), 'w') as f:
f.write(self.implementationContent)
def GenerateBaseGetLocationsFunction(self, ASTClassNames, CladeName):
MethodReturnType = 'NodeLocationAccessors'
Signature = \
'GetLocations(clang::{0} const *Object)'.format(CladeName)
ImplSignature = \
"""
GetLocationsImpl(std::shared_ptr<LocationCall> const& Prefix,
clang::{0} const *Object, SourceLocationMap &Locs,
SourceRangeMap &Rngs)
""".format(CladeName)
self.implementationContent += \
'void {0} {{ GetLocations{1}(Prefix, *Object, Locs, Rngs);'.format(
ImplSignature,
CladeName)
for ASTClassName in ASTClassNames:
if ASTClassName != CladeName:
self.implementationContent += \
"""
if (auto Derived = llvm::dyn_cast<clang::{0}>(Object)) {{
GetLocations{0}(Prefix, *Derived, Locs, Rngs);
}}
""".format(ASTClassName)
self.implementationContent += '}'
self.implementationContent += \
"""
{0} NodeIntrospection::{1} {{
NodeLocationAccessors Result;
std::shared_ptr<LocationCall> Prefix;
GetLocationsImpl(Prefix, Object, Result.LocationAccessors,
Result.RangeAccessors);
""".format(MethodReturnType,
Signature)
self.implementationContent += 'return Result; }'
def GenerateDynNodeVisitor(self, CladeNames):
MethodReturnType = 'NodeLocationAccessors'
Signature = \
'GetLocations(clang::DynTypedNode const &Node)'
self.implementationContent += MethodReturnType \
+ ' NodeIntrospection::' + Signature + '{'
for CladeName in CladeNames:
self.implementationContent += \
"""
if (const auto *N = Node.get<{0}>())
return GetLocations(const_cast<{0} *>(N));""".format(CladeName)
self.implementationContent += '\nreturn {}; }'
def GenerateEpilogue(self):
self.implementationContent += '''
}
}
'''
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--json-input-path',
help='Read API description from FILE', metavar='FILE')
parser.add_argument('--output-file', help='Generate output in FILEPATH',
metavar='FILEPATH')
parser.add_argument('--empty-implementation',
help='Generate empty implementation',
action="store", type=int)
options = parser.parse_args()
if options.empty_implementation:
with open(os.path.join(os.getcwd(),
options.output_file), 'w') as f:
f.write("""
namespace clang {
namespace tooling {
NodeLocationAccessors NodeIntrospection::GetLocations(clang::Stmt const *) {
return {};
}
NodeLocationAccessors
NodeIntrospection::GetLocations(clang::DynTypedNode const &) {
return {};
}
} // namespace tooling
} // namespace clang
""")
sys.exit(0)
with open(options.json_input_path) as f:
jsonData = json.load(f)
g = Generator()
g.GeneratePrologue()
if 'classesInClade' in jsonData:
for (CladeName, ClassNameData) in jsonData['classesInClade'].items():
g.GenerateBaseGetLocationsDeclaration(CladeName)
for (ClassName, ClassAccessors) in jsonData['classEntries'].items():
if ClassAccessors:
g.GenerateSrcLocMethod(ClassName, ClassAccessors)
for (CladeName, ClassNameData) in jsonData['classesInClade'].items():
g.GenerateBaseGetLocationsFunction(ClassNameData, CladeName)
g.GenerateDynNodeVisitor(jsonData['classesInClade'].keys())
g.GenerateEpilogue()
g.GenerateFiles(options.output_file)
if __name__ == '__main__':
main()

View File

@ -0,0 +1,61 @@
//===- NodeIntrospection.h -----------------------------------*- C++ -*----===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file contains the implementation of the NodeIntrospection.
//
//===----------------------------------------------------------------------===//
#include "clang/Tooling/NodeIntrospection.h"
#include "clang/AST/AST.h"
namespace clang {
namespace tooling {
std::string LocationCallFormatterCpp::format(LocationCall *Call) {
SmallVector<LocationCall *> vec;
while (Call) {
vec.push_back(Call);
Call = Call->on();
}
std::string result;
for (auto *VecCall : llvm::reverse(llvm::makeArrayRef(vec).drop_front())) {
result +=
(VecCall->name() + "()" + (VecCall->returnsPointer() ? "->" : "."))
.str();
}
result += (vec.back()->name() + "()").str();
return result;
}
namespace internal {
bool RangeLessThan::operator()(
std::pair<SourceRange, std::shared_ptr<LocationCall>> const &LHS,
std::pair<SourceRange, std::shared_ptr<LocationCall>> const &RHS) const {
if (!LHS.first.isValid() || !RHS.first.isValid())
return false;
if (LHS.first.getBegin() < RHS.first.getBegin())
return true;
else if (LHS.first.getBegin() != RHS.first.getBegin())
return false;
if (LHS.first.getEnd() < RHS.first.getEnd())
return true;
else if (LHS.first.getEnd() != RHS.first.getEnd())
return false;
return LHS.second->name() < RHS.second->name();
}
} // namespace internal
} // namespace tooling
} // namespace clang
#include "clang/Tooling/NodeIntrospection.inc"

View File

@ -29,6 +29,7 @@ add_subdirectory(ASTMatchers)
add_subdirectory(AST)
add_subdirectory(CrossTU)
add_subdirectory(Tooling)
add_subdirectory(Introspection)
add_subdirectory(Format)
add_subdirectory(Frontend)
add_subdirectory(Rewrite)

View File

@ -0,0 +1,25 @@
set(LLVM_LINK_COMPONENTS
FrontendOpenMP
Support
)
add_clang_unittest(IntrospectionTests
IntrospectionTest.cpp
)
clang_target_link_libraries(IntrospectionTests
PRIVATE
clangAST
clangASTMatchers
clangTooling
clangBasic
clangSerialization
clangFrontend
)
target_compile_definitions(IntrospectionTests PRIVATE
SKIP_INTROSPECTION_GENERATION=$<OR:$<CONFIG:Debug>,$<NOT:$<BOOL:${CLANG_TOOLING_BUILD_AST_INTROSPECTION}>>>
)
target_link_libraries(IntrospectionTests
PRIVATE
LLVMTestingSupport
)

View File

@ -0,0 +1,100 @@
//===- unittest/Introspection/IntrospectionTest.cpp ----------*- C++ -*---===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Tests for AST location API introspection.
//
//===----------------------------------------------------------------------===//
#include "clang/AST/ASTContext.h"
#include "clang/ASTMatchers/ASTMatchFinder.h"
#include "clang/ASTMatchers/ASTMatchers.h"
#include "clang/Tooling/NodeIntrospection.h"
#include "clang/Tooling/Tooling.h"
#include "gmock/gmock-matchers.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
using namespace clang;
using namespace clang::ast_matchers;
using namespace clang::tooling;
using ::testing::UnorderedElementsAre;
using ::testing::Pair;
#if SKIP_INTROSPECTION_GENERATION
TEST(Introspection, NonFatalAPI) {
auto AST = buildASTFromCode("void foo() {} void bar() { foo(); }", "foo.cpp",
std::make_shared<PCHContainerOperations>());
auto &Ctx = AST->getASTContext();
auto &TU = *Ctx.getTranslationUnitDecl();
auto BoundNodes = ast_matchers::match(
decl(hasDescendant(
callExpr(callee(functionDecl(hasName("foo")))).bind("fooCall"))),
TU, Ctx);
EXPECT_EQ(BoundNodes.size(), 1u);
auto *FooCall = BoundNodes[0].getNodeAs<CallExpr>("fooCall");
auto result = NodeIntrospection::GetLocations(FooCall);
EXPECT_EQ(result.LocationAccessors.size(), 0u);
EXPECT_EQ(result.RangeAccessors.size(), 0u);
}
#else
TEST(Introspection, SourceLocations) {
auto AST = buildASTFromCode("void foo() {} void bar() { foo(); }", "foo.cpp",
std::make_shared<PCHContainerOperations>());
auto &Ctx = AST->getASTContext();
auto &TU = *Ctx.getTranslationUnitDecl();
auto BoundNodes = ast_matchers::match(
decl(hasDescendant(
callExpr(callee(functionDecl(hasName("foo")))).bind("fooCall"))),
TU, Ctx);
EXPECT_EQ(BoundNodes.size(), 1u);
auto *FooCall = BoundNodes[0].getNodeAs<CallExpr>("fooCall");
auto result = NodeIntrospection::GetLocations(FooCall);
std::map<std::string, SourceLocation> ExpectedLocations;
llvm::transform(result.LocationAccessors,
std::inserter(ExpectedLocations, ExpectedLocations.end()),
[](const auto &Accessor) {
return std::make_pair(
LocationCallFormatterCpp::format(Accessor.second.get()),
Accessor.first);
});
EXPECT_THAT(ExpectedLocations,
UnorderedElementsAre(
Pair("getBeginLoc()", FooCall->getBeginLoc()),
Pair("getEndLoc()", FooCall->getEndLoc()),
Pair("getExprLoc()", FooCall->getExprLoc()),
Pair("getRParenLoc()", FooCall->getRParenLoc())));
std::map<std::string, SourceRange> ExpectedRanges;
llvm::transform(result.RangeAccessors,
std::inserter(ExpectedRanges, ExpectedRanges.end()),
[](const auto &Accessor) {
return std::make_pair(
LocationCallFormatterCpp::format(Accessor.second.get()),
Accessor.first);
});
EXPECT_THAT(ExpectedRanges,
UnorderedElementsAre(
Pair("getSourceRange()", FooCall->getSourceRange())));
}
#endif

View File

@ -1,7 +1,19 @@
# FIXME: The cmake build runs DumpTool:clang-ast-dump to generate a json
# file and feeds it into this step in non-debug builds or if an option is set.
action("node_introspection_inc") {
script = "DumpTool/generate_cxx_src_locs.py"
outputs = [ "$target_gen_dir/clang/Tooling/NodeIntrospection.inc" ]
args = [
"--empty-implementation=1",
"--output-file=" + rebase_path(outputs[0], root_build_dir),
]
}
static_library("Tooling") {
output_name = "clangTooling"
configs += [ "//llvm/utils/gn/build:clang_code" ]
deps = [
":node_introspection_inc",
"//clang/include/clang/Driver:Options",
"//clang/lib/AST",
"//clang/lib/ASTMatchers",
@ -13,6 +25,7 @@ static_library("Tooling") {
"//clang/lib/Rewrite",
"//clang/lib/Tooling/Core",
]
include_dirs = [ target_gen_dir ]
sources = [
"AllTUsExecution.cpp",
"ArgumentsAdjusters.cpp",
@ -25,6 +38,7 @@ static_library("Tooling") {
"GuessTargetAndModeCompilationDatabase.cpp",
"InterpolatingCompilationDatabase.cpp",
"JSONCompilationDatabase.cpp",
"NodeIntrospection.cpp",
"Refactoring.cpp",
"RefactoringCallbacks.cpp",
"StandaloneExecution.cpp",

View File

@ -0,0 +1,20 @@
executable("clang-ast-dump") {
configs += [ "//llvm/utils/gn/build:clang_code" ]
deps = [
"//clang/lib/AST",
"//clang/lib/ASTMatchers",
"//clang/lib/Basic",
"//clang/lib/Driver",
"//clang/lib/Format",
"//clang/lib/Frontend",
"//clang/lib/Lex",
"//clang/lib/Rewrite",
"//clang/lib/Serialization",
"//clang/lib/Tooling/Core",
]
sources = [
"ASTSrcLocProcessor.cpp",
"ClangSrcLocDump.cpp",
]
}

View File

@ -12,6 +12,7 @@ group("unittests") {
"Format:FormatTests",
"Frontend:FrontendTests",
"Index:IndexTests",
"Introspection:IntrospectionTests",
"Lex:LexTests",
"Rename:ClangRenameTests",
"Rewrite:RewriteTests",

View File

@ -0,0 +1,20 @@
import("//llvm/utils/unittest/unittest.gni")
unittest("IntrospectionTests") {
configs += [ "//llvm/utils/gn/build:clang_code" ]
deps = [
"//clang/lib/AST",
"//clang/lib/ASTMatchers",
"//clang/lib/Basic",
"//clang/lib/Frontend",
"//clang/lib/Serialization",
"//clang/lib/Tooling",
"//llvm/lib/Support",
"//llvm/lib/Testing/Support",
]
defines = [ "SKIP_INTROSPECTION_GENERATION" ]
sources = [ "IntrospectionTest.cpp" ]
}