[clangd] Store hash of command line in index shards.

Summary: This is to enable cache invalidation when command line flags changes.

Reviewers: sammccall

Subscribers: ilya-biryukov, MaskRay, jkorous, arphaman, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D64018

llvm-svn: 365121

clang-tools-extra/clangd/index/Background.cpp

@@ -25,6 +25,7 @@
 #include "index/SymbolCollector.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
+#include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/StringMap.h"
@@ -373,6 +374,11 @@ void BackgroundIndex::update(llvm::StringRef MainFile, IndexFileIn Index,
       Shard.Relations = RelS.get();
       Shard.Sources = IG.get();
 
+      // Only store command line hash for main files of the TU, since our
+      // current model keeps only one version of a header file.
+      if (Path == MainFile)
+        Shard.Cmd = Index.Cmd.getPointer();
+
       if (auto Error = IndexStorage->storeShard(Path, Shard))
         elog("Failed to write background-index shard for file {0}: {1}", Path,
              std::move(Error));
@@ -479,6 +485,7 @@ llvm::Error BackgroundIndex::index(tooling::CompileCommand Cmd,
 
   Action->EndSourceFile();
 
+  Index.Cmd = Inputs.CompileCommand;
   assert(Index.Symbols && Index.Refs && Index.Sources &&
          "Symbols, Refs and Sources must be set.");
 

clang-tools-extra/clangd/index/Serialization.cpp

@@ -13,9 +13,13 @@
 #include "SymbolOrigin.h"
 #include "Trace.h"
 #include "dex/Dex.h"
+#include "clang/Tooling/CompilationDatabase.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/raw_ostream.h"
+#include <vector>
 
 namespace clang {
 namespace clangd {
@@ -403,6 +407,30 @@ Relation readRelation(Reader &Data) {
   return {Subject, Predicate, Object};
 }
 
+struct InternedCompileCommand {
+  llvm::StringRef Directory;
+  std::vector<llvm::StringRef> CommandLine;
+};
+
+void writeCompileCommand(const InternedCompileCommand &Cmd,
+                         const StringTableOut &Strings,
+                         llvm::raw_ostream &CmdOS) {
+  writeVar(Strings.index(Cmd.Directory), CmdOS);
+  writeVar(Cmd.CommandLine.size(), CmdOS);
+  for (llvm::StringRef C : Cmd.CommandLine)
+    writeVar(Strings.index(C), CmdOS);
+}
+
+InternedCompileCommand
+readCompileCommand(Reader CmdReader, llvm::ArrayRef<llvm::StringRef> Strings) {
+  InternedCompileCommand Cmd;
+  Cmd.Directory = CmdReader.consumeString(Strings);
+  Cmd.CommandLine.resize(CmdReader.consumeVar());
+  for (llvm::StringRef &C : Cmd.CommandLine)
+    C = CmdReader.consumeString(Strings);
+  return Cmd;
+}
+
 // FILE ENCODING
 // A file is a RIFF chunk with type 'CdIx'.
 // It contains the sections:
@@ -490,6 +518,18 @@ llvm::Expected<IndexFileIn> readRIFF(llvm::StringRef Data) {
       return makeError("malformed or truncated relations");
     Result.Relations = std::move(Relations).build();
   }
+  if (Chunks.count("cmdl")) {
+    Reader CmdReader(Chunks.lookup("cmdl"));
+    if (CmdReader.err())
+      return makeError("malformed or truncated commandline section");
+    InternedCompileCommand Cmd =
+        readCompileCommand(CmdReader, Strings->Strings);
+    Result.Cmd.emplace();
+    Result.Cmd->Directory = Cmd.Directory;
+    Result.Cmd->CommandLine.reserve(Cmd.CommandLine.size());
+    for (llvm::StringRef C : Cmd.CommandLine)
+      Result.Cmd->CommandLine.emplace_back(C);
+  }
   return std::move(Result);
 }
 
@@ -547,6 +587,17 @@ void writeRIFF(const IndexFileOut &Data, llvm::raw_ostream &OS) {
     }
   }
 
+  InternedCompileCommand InternedCmd;
+  if (Data.Cmd) {
+    InternedCmd.CommandLine.reserve(Data.Cmd->CommandLine.size());
+    InternedCmd.Directory = Data.Cmd->Directory;
+    Strings.intern(InternedCmd.Directory);
+    for (llvm::StringRef C : Data.Cmd->CommandLine) {
+      InternedCmd.CommandLine.emplace_back(C);
+      Strings.intern(InternedCmd.CommandLine.back());
+    }
+  }
+
   std::string StringSection;
   {
     llvm::raw_string_ostream StringOS(StringSection);
@@ -592,6 +643,15 @@ void writeRIFF(const IndexFileOut &Data, llvm::raw_ostream &OS) {
     RIFF.Chunks.push_back({riff::fourCC("srcs"), SrcsSection});
   }
 
+  std::string CmdlSection;
+  if (Data.Cmd) {
+    {
+      llvm::raw_string_ostream CmdOS(CmdlSection);
+      writeCompileCommand(InternedCmd, Strings, CmdOS);
+    }
+    RIFF.Chunks.push_back({riff::fourCC("cmdl"), CmdlSection});
+  }
+
   OS << RIFF;
 }
 

clang-tools-extra/clangd/index/Serialization.h

@@ -27,6 +27,7 @@
 #include "Headers.h"
 #include "Index.h"
 #include "index/Symbol.h"
+#include "clang/Tooling/CompilationDatabase.h"
 #include "llvm/Support/Error.h"
 
 namespace clang {
@@ -44,6 +45,8 @@ struct IndexFileIn {
   llvm::Optional<RelationSlab> Relations;
   // Keys are URIs of the source files.
   llvm::Optional<IncludeGraph> Sources;
+  // This contains only the Directory and CommandLine.
+  llvm::Optional<tooling::CompileCommand> Cmd;
 };
 // Parse an index file. The input must be a RIFF or YAML file.
 llvm::Expected<IndexFileIn> readIndexFile(llvm::StringRef);
@@ -57,13 +60,15 @@ struct IndexFileOut {
   const IncludeGraph *Sources = nullptr;
   // TODO: Support serializing Dex posting lists.
   IndexFileFormat Format = IndexFileFormat::RIFF;
+  const tooling::CompileCommand *Cmd = nullptr;
 
   IndexFileOut() = default;
   IndexFileOut(const IndexFileIn &I)
       : Symbols(I.Symbols ? I.Symbols.getPointer() : nullptr),
         Refs(I.Refs ? I.Refs.getPointer() : nullptr),
         Relations(I.Relations ? I.Relations.getPointer() : nullptr),
-        Sources(I.Sources ? I.Sources.getPointer() : nullptr) {}
+        Sources(I.Sources ? I.Sources.getPointer() : nullptr),
+        Cmd(I.Cmd ? I.Cmd.getPointer() : nullptr) {}
 };
 // Serializes an index file.
 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const IndexFileOut &O);

clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp

@@ -2,6 +2,7 @@
 #include "TestFS.h"
 #include "TestTU.h"
 #include "index/Background.h"
+#include "clang/Tooling/CompilationDatabase.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include "llvm/Support/Threading.h"
 #include "gmock/gmock.h"
@@ -526,5 +527,49 @@ TEST_F(BackgroundIndexTest, UncompilableFiles) {
                                    "unittest:///B.h"));
 }
 
+TEST_F(BackgroundIndexTest, CmdLineHash) {
+  MockFSProvider FS;
+  llvm::StringMap<std::string> Storage;
+  size_t CacheHits = 0;
+  MemoryShardStorage MSS(Storage, CacheHits);
+  OverlayCDB CDB(/*Base=*/nullptr, /*FallbackFlags=*/{},
+                 /*ResourceDir=*/std::string(""));
+  BackgroundIndex Idx(Context::empty(), FS, CDB,
+                      [&](llvm::StringRef) { return &MSS; });
+
+  tooling::CompileCommand Cmd;
+  FS.Files[testPath("A.cc")] = "#include \"A.h\"";
+  FS.Files[testPath("A.h")] = "";
+  Cmd.Filename = "../A.cc";
+  Cmd.Directory = testPath("build");
+  Cmd.CommandLine = {"clang++", "../A.cc", "-fsyntax-only"};
+  CDB.setCompileCommand(testPath("build/../A.cc"), Cmd);
+  ASSERT_TRUE(Idx.blockUntilIdleForTest());
+
+  EXPECT_THAT(Storage.keys(), ElementsAre(testPath("A.cc"), testPath("A.h")));
+  // Make sure we only store the Cmd for main file.
+  EXPECT_FALSE(MSS.loadShard(testPath("A.h"))->Cmd);
+
+  {
+    tooling::CompileCommand CmdStored = *MSS.loadShard(testPath("A.cc"))->Cmd;
+    EXPECT_EQ(CmdStored.CommandLine, Cmd.CommandLine);
+    EXPECT_EQ(CmdStored.Directory, Cmd.Directory);
+  }
+
+  // FIXME: Changing compile commands should be enough to invalidate the cache.
+  FS.Files[testPath("A.cc")] = " ";
+  Cmd.CommandLine = {"clang++", "../A.cc", "-Dfoo", "-fsyntax-only"};
+  CDB.setCompileCommand(testPath("build/../A.cc"), Cmd);
+  ASSERT_TRUE(Idx.blockUntilIdleForTest());
+
+  EXPECT_FALSE(MSS.loadShard(testPath("A.h"))->Cmd);
+
+  {
+    tooling::CompileCommand CmdStored = *MSS.loadShard(testPath("A.cc"))->Cmd;
+    EXPECT_EQ(CmdStored.CommandLine, Cmd.CommandLine);
+    EXPECT_EQ(CmdStored.Directory, Cmd.Directory);
+  }
+}
+
 } // namespace clangd
 } // namespace clang

clang-tools-extra/clangd/unittests/SerializationTests.cpp

@@ -8,6 +8,7 @@
 
 #include "index/Index.h"
 #include "index/Serialization.h"
+#include "clang/Tooling/CompilationDatabase.h"
 #include "llvm/Support/SHA1.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include "gmock/gmock.h"
@@ -240,6 +241,36 @@ TEST(SerializationTest, SrcsTest) {
   }
 }
 
+TEST(SerializationTest, CmdlTest) {
+  auto In = readIndexFile(YAML);
+  EXPECT_TRUE(bool(In)) << In.takeError();
+
+  tooling::CompileCommand Cmd;
+  Cmd.Directory = "testdir";
+  Cmd.CommandLine.push_back("cmd1");
+  Cmd.CommandLine.push_back("cmd2");
+  Cmd.Filename = "ignored";
+  Cmd.Heuristic = "ignored";
+  Cmd.Output = "ignored";
+
+  IndexFileOut Out(*In);
+  Out.Format = IndexFileFormat::RIFF;
+  Out.Cmd = &Cmd;
+  {
+    std::string Serialized = llvm::to_string(Out);
+
+    auto In = readIndexFile(Serialized);
+    ASSERT_TRUE(bool(In)) << In.takeError();
+    ASSERT_TRUE(In->Cmd);
+
+    const tooling::CompileCommand &SerializedCmd = In->Cmd.getValue();
+    EXPECT_EQ(SerializedCmd.CommandLine, Cmd.CommandLine);
+    EXPECT_EQ(SerializedCmd.Directory, Cmd.Directory);
+    EXPECT_NE(SerializedCmd.Filename, Cmd.Filename);
+    EXPECT_NE(SerializedCmd.Heuristic, Cmd.Heuristic);
+    EXPECT_NE(SerializedCmd.Output, Cmd.Output);
+  }
+}
 } // namespace
 } // namespace clangd
 } // namespace clang