[PCH] Allow VFS to be used for tests that generate PCH files

When using a virtual file-system (VFS) and a preamble file (PCH) is generated,
it is generated on-disk in the real file-system instead of in the VFS (which
makes sense, since the VFS is read-only). However, when subsequently reading
the generated PCH, the frontend passes through the VFS it has been given --
resulting in an error and a failed parse (since the VFS doesn't contain the
PCH; the real filesystem does).

This patch fixes that by detecting when a VFS is being used for a parse that
needs to work with a PCH file, and creating an overlay VFS that includes the
PCH file from the real file-system.

This allows tests to be written which make use of both PCH files and a VFS.

Differential Revision: https://reviews.llvm.org/D37474

llvm-svn: 312917

clang/include/clang/Frontend/PrecompiledPreamble.h

@@ -97,9 +97,12 @@ public:
   PrecompiledPreamble(PrecompiledPreamble &&) = default;
   PrecompiledPreamble &operator=(PrecompiledPreamble &&) = default;
 
-  /// PreambleBounds used to build the preamble
+  /// PreambleBounds used to build the preamble.
   PreambleBounds getBounds() const;
 
+  /// The temporary file path at which the preamble PCH was placed.
+  StringRef GetPCHPath() const { return PCHFile.getFilePath(); }
+
   /// Check whether PrecompiledPreamble can be reused for the new contents(\p
   /// MainFileBuffer) of the main file.
   bool CanReuse(const CompilerInvocation &Invocation,

clang/lib/Frontend/ASTUnit.cpp

@@ -1009,6 +1009,24 @@ static void checkAndSanitizeDiags(SmallVectorImpl<StoredDiagnostic> &
   }
 }
 
+static IntrusiveRefCntPtr<vfs::FileSystem> createVFSOverlayForPreamblePCH(
+    StringRef PCHFilename,
+    IntrusiveRefCntPtr<vfs::FileSystem> RealFS,
+    IntrusiveRefCntPtr<vfs::FileSystem> VFS) {
+  // We want only the PCH file from the real filesystem to be available,
+  // so we create an in-memory VFS with just that and overlay it on top.
+  auto Buf = RealFS->getBufferForFile(PCHFilename);
+  if (!Buf)
+    return VFS;
+  IntrusiveRefCntPtr<vfs::InMemoryFileSystem>
+      PCHFS(new vfs::InMemoryFileSystem());
+  PCHFS->addFile(PCHFilename, 0, std::move(*Buf));
+  IntrusiveRefCntPtr<vfs::OverlayFileSystem>
+      Overlay(new vfs::OverlayFileSystem(VFS));
+  Overlay->pushOverlay(PCHFS);
+  return Overlay;
+}
+
 /// Parse the source file into a translation unit using the given compiler
 /// invocation, replacing the current translation unit.
 ///
@@ -1030,6 +1048,24 @@ bool ASTUnit::Parse(std::shared_ptr<PCHContainerOperations> PCHContainerOps,
     Clang->setVirtualFileSystem(VFS);
   }
 
+  // Make sure we can access the PCH file even if we're using a VFS
+  if (!VFS && FileMgr)
+    VFS = FileMgr->getVirtualFileSystem();
+  IntrusiveRefCntPtr<vfs::FileSystem> RealFS = vfs::getRealFileSystem();
+  if (OverrideMainBuffer && VFS && RealFS && VFS != RealFS &&
+      !VFS->exists(Preamble->GetPCHPath())) {
+    // We have a slight inconsistency here -- we're using the VFS to
+    // read files, but the PCH was generated in the real file system.
+    VFS = createVFSOverlayForPreamblePCH(Preamble->GetPCHPath(), RealFS, VFS);
+    if (FileMgr) {
+      FileMgr = new FileManager(FileMgr->getFileSystemOpts(), VFS);
+      Clang->setFileManager(FileMgr.get());
+    }
+    else {
+      Clang->setVirtualFileSystem(VFS);
+    }
+  }
+
   // Recover resources if we crash before exiting this method.
   llvm::CrashRecoveryContextCleanupRegistrar<CompilerInstance>
     CICleanup(Clang.get());

clang/unittests/Frontend/CMakeLists.txt

@@ -6,6 +6,7 @@ add_clang_unittest(FrontendTests
   ASTUnitTest.cpp
   FrontendActionTest.cpp
   CodeGenActionTest.cpp
+  PCHPreambleTest.cpp
   )
 target_link_libraries(FrontendTests
   clangAST

clang/unittests/Frontend/PCHPreambleTest.cpp

@@ -0,0 +1,156 @@
+//====-- unittests/Frontend/PCHPreambleTest.cpp - FrontendAction tests ---====//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Frontend/ASTUnit.h"
+#include "clang/Frontend/CompilerInvocation.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/FrontendActions.h"
+#include "clang/Frontend/FrontendOptions.h"
+#include "clang/Lex/PreprocessorOptions.h"
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/FileManager.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace clang;
+
+namespace {
+
+class ReadCountingInMemoryFileSystem : public vfs::InMemoryFileSystem
+{
+  std::map<std::string, unsigned> ReadCounts;
+
+public:
+  ErrorOr<std::unique_ptr<vfs::File>> openFileForRead(const Twine &Path) override
+  {
+    SmallVector<char, 128> PathVec;
+    Path.toVector(PathVec);
+    llvm::sys::path::remove_dots(PathVec, true);
+    ++ReadCounts[std::string(PathVec.begin(), PathVec.end())];
+    return InMemoryFileSystem::openFileForRead(Path);
+  }
+
+  unsigned GetReadCount(const Twine &Path) const
+  {
+    auto it = ReadCounts.find(Path.str());
+    return it == ReadCounts.end() ? 0 : it->second;
+  }
+};
+
+class PCHPreambleTest : public ::testing::Test {
+  IntrusiveRefCntPtr<ReadCountingInMemoryFileSystem> VFS;
+  StringMap<std::string> RemappedFiles;
+  std::shared_ptr<PCHContainerOperations> PCHContainerOpts;
+  FileSystemOptions FSOpts;
+
+public:
+  void SetUp() override {
+    VFS = new ReadCountingInMemoryFileSystem();
+    // We need the working directory to be set to something absolute,
+    // otherwise it ends up being inadvertently set to the current
+    // working directory in the real file system due to a series of
+    // unfortunate conditions interacting badly.
+    // What's more, this path *must* be absolute on all (real)
+    // filesystems, so just '/' won't work (e.g. on Win32).
+    VFS->setCurrentWorkingDirectory("//./");
+  }
+
+  void TearDown() override {
+  }
+
+  void AddFile(const std::string &Filename, const std::string &Contents) {
+    ::time_t now;
+    ::time(&now);
+    VFS->addFile(Filename, now, MemoryBuffer::getMemBufferCopy(Contents, Filename));
+  }
+
+  void RemapFile(const std::string &Filename, const std::string &Contents) {
+    RemappedFiles[Filename] = Contents;
+  }
+
+  std::unique_ptr<ASTUnit> ParseAST(const std::string &EntryFile) {
+    PCHContainerOpts = std::make_shared<PCHContainerOperations>();
+    std::shared_ptr<CompilerInvocation> CI(new CompilerInvocation);
+    CI->getFrontendOpts().Inputs.push_back(
+      FrontendInputFile(EntryFile, FrontendOptions::getInputKindForExtension(
+        llvm::sys::path::extension(EntryFile).substr(1))));
+
+    CI->getTargetOpts().Triple = "i386-unknown-linux-gnu";
+
+    CI->getPreprocessorOpts().RemappedFileBuffers = GetRemappedFiles();
+
+    PreprocessorOptions &PPOpts = CI->getPreprocessorOpts();
+    PPOpts.RemappedFilesKeepOriginalName = true;
+
+    IntrusiveRefCntPtr<DiagnosticsEngine>
+      Diags(CompilerInstance::createDiagnostics(new DiagnosticOptions, new DiagnosticConsumer));
+
+    FileManager *FileMgr = new FileManager(FSOpts, VFS);
+
+    std::unique_ptr<ASTUnit> AST = ASTUnit::LoadFromCompilerInvocation(
+      CI, PCHContainerOpts, Diags, FileMgr, false, false,
+      /*PrecompilePreambleAfterNParses=*/1);
+    return AST;
+  }
+
+  bool ReparseAST(const std::unique_ptr<ASTUnit> &AST) {
+    bool reparseFailed = AST->Reparse(PCHContainerOpts, GetRemappedFiles(), VFS);
+    return !reparseFailed;
+  }
+
+  unsigned GetFileReadCount(const std::string &Filename) const {
+    return VFS->GetReadCount(Filename);
+  }
+
+private:
+  std::vector<std::pair<std::string, llvm::MemoryBuffer *>>
+  GetRemappedFiles() const {
+    std::vector<std::pair<std::string, llvm::MemoryBuffer *>> Remapped;
+    for (const auto &RemappedFile : RemappedFiles) {
+      std::unique_ptr<MemoryBuffer> buf = MemoryBuffer::getMemBufferCopy(
+        RemappedFile.second, RemappedFile.first());
+      Remapped.emplace_back(RemappedFile.first(), buf.release());
+    }
+    return Remapped;
+  }
+};
+
+TEST_F(PCHPreambleTest, ReparseWithOverriddenFileDoesNotInvalidatePreamble) {
+  std::string Header1 = "//./header1.h";
+  std::string Header2 = "//./header2.h";
+  std::string MainName = "//./main.cpp";
+  AddFile(Header1, "");
+  AddFile(Header2, "#pragma once");
+  AddFile(MainName,
+    "#include \"//./foo/../header1.h\"\n"
+    "#include \"//./foo/../header2.h\"\n"
+    "int main() { return ZERO; }");
+  RemapFile(Header1, "static const int ZERO = 0;\n");
+
+  std::unique_ptr<ASTUnit> AST(ParseAST(MainName));
+  ASSERT_TRUE(AST.get());
+  ASSERT_FALSE(AST->getDiagnostics().hasErrorOccurred());
+
+  unsigned initialCounts[] = {
+    GetFileReadCount(MainName),
+    GetFileReadCount(Header1),
+    GetFileReadCount(Header2)
+  };
+
+  ASSERT_TRUE(ReparseAST(AST));
+
+  ASSERT_NE(initialCounts[0], GetFileReadCount(MainName));
+  ASSERT_EQ(initialCounts[1], GetFileReadCount(Header1));
+  ASSERT_EQ(initialCounts[2], GetFileReadCount(Header2));
+}
+
+} // anonymous namespace