[clang-extdef-mapping] Directly process .ast files

When doing CTU analysis setup you pre-compile .cpp to .ast and then
you run clang-extdef-mapping on the .cpp file as well. This is a
pretty slow process since we have to recompile the file each time.

With this patch you can now run clang-extdef-mapping directly on
the .ast file. That saves a lot of time.

I tried this on llvm/lib/AsmParser/Parser.cpp and running
extdef-mapping on the .cpp file took 5.4s on my machine.

While running it on the .ast file it took 2s.

This can save a lot of time for the setup phase of CTU analysis.

Reviewed By: martong

Differential Revision: https://reviews.llvm.org/D128704

clang/docs/ReleaseNotes.rst

@@ -587,6 +587,13 @@ clang-format
 - Option ``InsertBraces`` has been added to insert optional braces after control
   statements.
 
+clang-extdef-mapping
+--------------------
+
+- clang-extdef-mapping now accepts .ast files as input. This is faster than to
+  recompile the files from sources when extracting method definitons. This can
+  be really beneficial when creating .ast files for input to the clang-static-analyzer.
+
 libclang
 --------
 

clang/test/Analysis/func-mapping-test.cpp

@@ -1,4 +1,6 @@
 // RUN: %clang_extdef_map %s -- | FileCheck --implicit-check-not "c:@y" --implicit-check-not "c:@z" %s
+// RUN: %clang -emit-ast %s -o %t.ast
+// RUN: %clang_extdef_map %t.ast -- | FileCheck --implicit-check-not "c:@y" --implicit-check-not "c:@z" %s
 
 int f(int) {
   return 0;

clang/tools/clang-extdef-mapping/ClangExtDefMapGen.cpp

@@ -1,4 +1,4 @@
-//===- ClangExtDefMapGen.cpp -----------------------------------------------===//
+//===- ClangExtDefMapGen.cpp ---------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -13,10 +13,12 @@
 
 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/Basic/DiagnosticOptions.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/CrossTU/CrossTranslationUnit.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/FrontendActions.h"
+#include "clang/Frontend/TextDiagnosticPrinter.h"
 #include "clang/Tooling/CommonOptionsParser.h"
 #include "clang/Tooling/Tooling.h"
 #include "llvm/Support/CommandLine.h"
@@ -29,12 +31,16 @@ using namespace clang;
 using namespace clang::cross_tu;
 using namespace clang::tooling;
 
-static cl::OptionCategory ClangExtDefMapGenCategory("clang-extdefmapgen options");
+static cl::OptionCategory
+    ClangExtDefMapGenCategory("clang-extdefmapgen options");
 
 class MapExtDefNamesConsumer : public ASTConsumer {
 public:
-  MapExtDefNamesConsumer(ASTContext &Context)
-      : Ctx(Context), SM(Context.getSourceManager()) {}
+  MapExtDefNamesConsumer(ASTContext &Context,
+                         StringRef astFilePath = StringRef())
+      : Ctx(Context), SM(Context.getSourceManager()) {
+    CurrentFileName = astFilePath.str();
+  }
 
   ~MapExtDefNamesConsumer() {
     // Flush results to standard output.
@@ -111,6 +117,82 @@ protected:
 
 static cl::extrahelp CommonHelp(CommonOptionsParser::HelpMessage);
 
+static IntrusiveRefCntPtr<DiagnosticsEngine> Diags;
+
+IntrusiveRefCntPtr<DiagnosticsEngine> GetDiagnosticsEngine() {
+  if (Diags) {
+    // Call reset to make sure we don't mix errors
+    Diags->Reset(false);
+    return Diags;
+  }
+
+  IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
+  TextDiagnosticPrinter *DiagClient =
+      new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts);
+  DiagClient->setPrefix("clang-extdef-mappping");
+  IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
+
+  IntrusiveRefCntPtr<DiagnosticsEngine> DiagEngine(
+      new DiagnosticsEngine(DiagID, &*DiagOpts, DiagClient));
+  Diags.swap(DiagEngine);
+
+  // Retain this one time so it's not destroyed by ASTUnit::LoadFromASTFile
+  Diags->Retain();
+  return Diags;
+}
+
+static CompilerInstance *CI = nullptr;
+
+static bool HandleAST(StringRef AstPath) {
+
+  if (!CI)
+    CI = new CompilerInstance();
+
+  IntrusiveRefCntPtr<DiagnosticsEngine> DiagEngine = GetDiagnosticsEngine();
+
+  std::unique_ptr<ASTUnit> Unit = ASTUnit::LoadFromASTFile(
+      AstPath.str(), CI->getPCHContainerOperations()->getRawReader(),
+      ASTUnit::LoadASTOnly, DiagEngine, CI->getFileSystemOpts());
+
+  if (!Unit)
+    return false;
+
+  FileManager FM(CI->getFileSystemOpts());
+  SmallString<128> AbsPath(AstPath);
+  FM.makeAbsolutePath(AbsPath);
+
+  MapExtDefNamesConsumer Consumer =
+      MapExtDefNamesConsumer(Unit->getASTContext(), AbsPath);
+  Consumer.HandleTranslationUnit(Unit->getASTContext());
+
+  return true;
+}
+
+static int HandleFiles(ArrayRef<std::string> SourceFiles,
+                       CompilationDatabase &compilations) {
+  std::vector<std::string> SourcesToBeParsed;
+
+  // Loop over all input files, if they are pre-compiled AST
+  // process them directly in HandleAST, otherwise put them
+  // on a list for ClangTool to handle.
+  for (StringRef Src : SourceFiles) {
+    if (Src.endswith(".ast")) {
+      if (!HandleAST(Src)) {
+        return 1;
+      }
+    } else {
+      SourcesToBeParsed.push_back(Src.str());
+    }
+  }
+
+  if (!SourcesToBeParsed.empty()) {
+    ClangTool Tool(compilations, SourcesToBeParsed);
+    return Tool.run(newFrontendActionFactory<MapExtDefNamesAction>().get());
+  }
+
+  return 0;
+}
+
 int main(int argc, const char **argv) {
   // Print a stack trace if we signal out.
   sys::PrintStackTraceOnErrorSignal(argv[0], false);
@@ -118,7 +200,10 @@ int main(int argc, const char **argv) {
 
   const char *Overview = "\nThis tool collects the USR name and location "
                          "of external definitions in the source files "
-                         "(excluding headers).\n";
+                         "(excluding headers).\n"
+                         "Input can be either source files that are compiled "
+                         "with compile database or .ast files that are "
+                         "created from clang's -emit-ast option.\n";
   auto ExpectedParser = CommonOptionsParser::create(
       argc, argv, ClangExtDefMapGenCategory, cl::ZeroOrMore, Overview);
   if (!ExpectedParser) {
@@ -127,8 +212,6 @@ int main(int argc, const char **argv) {
   }
   CommonOptionsParser &OptionsParser = ExpectedParser.get();
 
-  ClangTool Tool(OptionsParser.getCompilations(),
-                 OptionsParser.getSourcePathList());
-
-  return Tool.run(newFrontendActionFactory<MapExtDefNamesAction>().get());
+  return HandleFiles(OptionsParser.getSourcePathList(),
+                     OptionsParser.getCompilations());
 }