Initial implementation of virtual file system

This adds the minimum virtual file system support to start migrating
FileManager onto the VFS.

Originally discussed here:
http://lists.cs.uiuc.edu/pipermail/cfe-dev/2014-February/035188.html

Differential Revision: http://llvm-reviews.chandlerc.com/D2745

llvm-svn: 201618

clang/include/clang/Basic/FileManager.h

@@ -17,6 +17,7 @@
 
 #include "clang/Basic/FileSystemOptions.h"
 #include "clang/Basic/LLVM.h"
+#include "clang/Basic/VirtualFileSystem.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/OwningPtr.h"
@@ -24,7 +25,6 @@
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Allocator.h"
-#include "llvm/Support/FileSystem.h"
 // FIXME: Enhance libsystem to support inode and other fields in stat.
 #include <sys/types.h>
 
@@ -55,7 +55,7 @@ public:
 /// \brief Cached information about one file (either on disk
 /// or in the virtual file system).
 ///
-/// If the 'FD' member is valid, then this FileEntry has an open file
+/// If the 'File' member is valid, then this FileEntry has an open file
 /// descriptor for the file.
 class FileEntry {
   const char *Name;           // Name of the file.
@@ -67,31 +67,33 @@ class FileEntry {
   bool IsNamedPipe;
   bool InPCH;
 
-  /// FD - The file descriptor for the file entry if it is opened and owned
+  /// \brief The open file, if it is owned by the \p FileEntry.
-  /// by the FileEntry.  If not, this is set to -1.
+  mutable OwningPtr<vfs::File> File;
-  mutable int FD;
   friend class FileManager;
 
+  void closeFile() const {
+    File.reset(0); // rely on destructor to close File
+  }
+
 public:
   FileEntry(llvm::sys::fs::UniqueID UniqueID, bool IsNamedPipe, bool InPCH)
-      : Name(0), UniqueID(UniqueID), IsNamedPipe(IsNamedPipe), InPCH(InPCH),
+      : Name(0), UniqueID(UniqueID), IsNamedPipe(IsNamedPipe), InPCH(InPCH)
-        FD(-1) {}
+  {}
   // Add a default constructor for use with llvm::StringMap
   FileEntry()
-      : Name(0), UniqueID(0, 0), IsNamedPipe(false), InPCH(false), FD(-1) {}
+      : Name(0), UniqueID(0, 0), IsNamedPipe(false), InPCH(false)
+  {}
 
   FileEntry(const FileEntry &FE) {
     memcpy(this, &FE, sizeof(FE));
-    assert(FD == -1 && "Cannot copy a file-owning FileEntry");
+    assert(!File && "Cannot copy a file-owning FileEntry");
   }
 
   void operator=(const FileEntry &FE) {
     memcpy(this, &FE, sizeof(FE));
-    assert(FD == -1 && "Cannot assign a file-owning FileEntry");
+    assert(!File && "Cannot assign a file-owning FileEntry");
   }
 
-  ~FileEntry();
-
   const char *getName() const { return Name; }
   off_t getSize() const { return Size; }
   unsigned getUID() const { return UID; }
@@ -119,6 +121,7 @@ struct FileData;
 /// as a single file.
 ///
 class FileManager : public RefCountedBase<FileManager> {
+  IntrusiveRefCntPtr<vfs::FileSystem> FS;
   FileSystemOptions FileSystemOpts;
 
   class UniqueDirContainer;
@@ -172,14 +175,15 @@ class FileManager : public RefCountedBase<FileManager> {
   OwningPtr<FileSystemStatCache> StatCache;
 
   bool getStatValue(const char *Path, FileData &Data, bool isFile,
-                    int *FileDescriptor);
+                    vfs::File **F);
 
   /// Add all ancestors of the given path (pointing to either a file
   /// or a directory) as virtual directories.
   void addAncestorsAsVirtualDirs(StringRef Path);
 
 public:
-  FileManager(const FileSystemOptions &FileSystemOpts);
+  FileManager(const FileSystemOptions &FileSystemOpts,
+              llvm::IntrusiveRefCntPtr<vfs::FileSystem> FS = 0);
   ~FileManager();
 
   /// \brief Installs the provided FileSystemStatCache object within
@@ -248,7 +252,7 @@ public:
   ///
   /// \returns false on success, true on error.
   bool getNoncachedStatValue(StringRef Path,
-                             llvm::sys::fs::file_status &Result);
+                             vfs::Status &Result);
 
   /// \brief Remove the real file \p Entry from the cache.
   void invalidateCache(const FileEntry *Entry);

clang/include/clang/Basic/FileSystemStatCache.h

@@ -24,6 +24,11 @@
 
 namespace clang {
 
+namespace vfs {
+class File;
+class FileSystem;
+}
+
 struct FileData {
   uint64_t Size;
   time_t ModTime;
@@ -57,10 +62,11 @@ public:
   /// If isFile is true, then this lookup should only return success for files
   /// (not directories).  If it is false this lookup should only return
   /// success for directories (not files).  On a successful file lookup, the
-  /// implementation can optionally fill in FileDescriptor with a valid
+  /// implementation can optionally fill in \p F with a valid \p File object and
-  /// descriptor and the client guarantees that it will close it.
+  /// the client guarantees that it will close it.
   static bool get(const char *Path, FileData &Data, bool isFile,
-                  int *FileDescriptor, FileSystemStatCache *Cache);
+                  vfs::File **F, FileSystemStatCache *Cache,
+                  vfs::FileSystem &FS);
 
   /// \brief Sets the next stat call cache in the chain of stat caches.
   /// Takes ownership of the given stat cache.
@@ -78,17 +84,16 @@ public:
   
 protected:
   virtual LookupResult getStat(const char *Path, FileData &Data, bool isFile,
-                               int *FileDescriptor) = 0;
+                               vfs::File **F, vfs::FileSystem &FS) = 0;
 
   LookupResult statChained(const char *Path, FileData &Data, bool isFile,
-                           int *FileDescriptor) {
+                           vfs::File **F, vfs::FileSystem &FS) {
     if (FileSystemStatCache *Next = getNextStatCache())
-      return Next->getStat(Path, Data, isFile, FileDescriptor);
+      return Next->getStat(Path, Data, isFile, F, FS);
 
     // If we hit the end of the list of stat caches to try, just compute and
     // return it without a cache.
-    return get(Path, Data, isFile, FileDescriptor, 0) ? CacheMissing
+    return get(Path, Data, isFile, F, 0, FS) ? CacheMissing : CacheExists;
-                                                      : CacheExists;
   }
 };
 
@@ -107,7 +112,7 @@ public:
   iterator end() const { return StatCalls.end(); }
 
   virtual LookupResult getStat(const char *Path, FileData &Data, bool isFile,
-                               int *FileDescriptor);
+                               vfs::File **F, vfs::FileSystem &FS);
 };
 
 } // end namespace clang

clang/include/clang/Basic/VirtualFileSystem.h

@@ -0,0 +1,127 @@
+//===- VirtualFileSystem.h - Virtual File System Layer ----------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+/// \file
+/// \brief Defines the virtual file system interface vfs::FileSystem.
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_BASIC_VIRTUAL_FILE_SYSTEM_H
+#define LLVM_CLANG_BASIC_VIRTUAL_FILE_SYSTEM_H
+
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/ErrorOr.h"
+
+namespace llvm {
+template <typename T> class OwningPtr;
+class MemoryBuffer;
+}
+
+namespace clang {
+namespace vfs {
+
+/// \brief The result of a \p status operation.
+class Status {
+  std::string Name;
+  std::string ExternalName;
+  llvm::sys::fs::UniqueID UID;
+  llvm::sys::TimeValue MTime;
+  uint32_t User;
+  uint32_t Group;
+  uint64_t Size;
+  llvm::sys::fs::file_type Type;
+  llvm::sys::fs::perms Perms;
+
+public:
+  Status() : Type(llvm::sys::fs::file_type::status_error) {}
+  Status(const llvm::sys::fs::file_status &Status);
+  Status(llvm::StringRef Name, llvm::StringRef RealName,
+         llvm::sys::fs::UniqueID UID, llvm::sys::TimeValue MTime,
+         uint32_t User, uint32_t Group, uint64_t Size,
+         llvm::sys::fs::file_type Type, llvm::sys::fs::perms Perms);
+
+  /// \brief Returns the name this status was looked up by.
+  llvm::StringRef getName() const { return Name; }
+
+  /// \brief Returns the name to use outside the compiler.
+  ///
+  /// For example, in diagnostics or debug info we should use this name.
+  llvm::StringRef getExternalName() const { return ExternalName; }
+
+  void setName(llvm::StringRef N) { Name = N; }
+  void setExternalName(llvm::StringRef N) { ExternalName = N; }
+
+  /// @name Status interface from llvm::sys::fs
+  /// @{
+  llvm::sys::fs::file_type getType() const { return Type; }
+  llvm::sys::fs::perms getPermissions() const { return Perms; }
+  llvm::sys::TimeValue getLastModificationTime() const { return MTime; }
+  llvm::sys::fs::UniqueID getUniqueID() const { return UID; }
+  uint32_t getUser() const { return User; }
+  uint32_t getGroup() const { return Group; }
+  uint64_t getSize() const { return Size; }
+  void setType(llvm::sys::fs::file_type v) { Type = v; }
+  void setPermissions(llvm::sys::fs::perms p) { Perms = p; }
+  /// @}
+  /// @name Status queries
+  /// These are static queries in llvm::sys::fs.
+  /// @{
+  bool equivalent(const Status &Other) const;
+  bool isDirectory() const;
+  bool isRegularFile() const;
+  bool isOther() const;
+  bool isSymlink() const;
+  bool isStatusKnown() const;
+  bool exists() const;
+  /// @}
+};
+
+/// \brief Represents an open file.
+class File {
+public:
+  /// \brief Destroy the file after closing it (if open).
+  /// Sub-classes should generally call close() inside their destructors.  We
+  /// cannot do that from the base class, since close is virtual.
+  virtual ~File();
+  /// \brief Get the status of the file.
+  virtual llvm::ErrorOr<Status> status() = 0;
+  /// \brief Get the contents of the file as a \p MemoryBuffer.
+  virtual llvm::error_code
+  getBuffer(const llvm::Twine &Name,
+            llvm::OwningPtr<llvm::MemoryBuffer> &Result, int64_t FileSize = -1,
+            bool RequiresNullTerminator = true) = 0;
+  /// \brief Closes the file.
+  virtual llvm::error_code close() = 0;
+};
+
+/// \brief The virtual file system interface.
+class FileSystem : public llvm::RefCountedBase<FileSystem> {
+public:
+  virtual ~FileSystem();
+
+  /// \brief Get the status of the entry at \p Path, if one exists.
+  virtual llvm::ErrorOr<Status> status(const llvm::Twine &Path) = 0;
+  /// \brief Get a \p File object for the file at \p Path, if one exists.
+  virtual llvm::error_code openFileForRead(const llvm::Twine &Path,
+                                           llvm::OwningPtr<File> &Result) = 0;
+
+  /// This is a convenience method that opens a file, gets its content and then
+  /// closes the file.
+  llvm::error_code getBufferForFile(const llvm::Twine &Name,
+                                    llvm::OwningPtr<llvm::MemoryBuffer> &Result,
+                                    int64_t FileSize = -1,
+                                    bool RequiresNullTerminator = true);
+};
+
+/// \brief Gets an \p vfs::FileSystem for the 'real' file system, as seen by
+/// the operating system.
+llvm::IntrusiveRefCntPtr<FileSystem> getRealFileSystem();
+
+} // end namespace vfs
+} // end namespace clang
+#endif // LLVM_CLANG_BASIC_VIRTUAL_FILE_SYSTEM_H

clang/include/clang/Frontend/CompilerInstance.h

@@ -75,6 +75,9 @@ class CompilerInstance : public ModuleLoader {
   /// The target being compiled for.
   IntrusiveRefCntPtr<TargetInfo> Target;
 
+  /// The virtual file system.
+  IntrusiveRefCntPtr<vfs::FileSystem> VirtualFileSystem;
+
   /// The file manager.
   IntrusiveRefCntPtr<FileManager> FileMgr;
 
@@ -313,6 +316,23 @@ public:
   /// Replace the current diagnostics engine.
   void setTarget(TargetInfo *Value);
 
+  /// }
+  /// @name Virtual File System
+  /// {
+
+  bool hasVirtualFileSystem() const { return VirtualFileSystem != 0; }
+
+  vfs::FileSystem &getVirtualFileSystem() const {
+    assert(hasVirtualFileSystem() &&
+           "Compiler instance has no virtual file system");
+    return *VirtualFileSystem;
+  }
+
+  /// \brief Replace the current virtual file system.
+  void setVirtualFileSystem(IntrusiveRefCntPtr<vfs::FileSystem> FS) {
+    VirtualFileSystem = FS;
+  }
+
   /// }
   /// @name File Manager
   /// {
@@ -527,6 +547,10 @@ public:
                     bool ShouldOwnClient = true,
                     const CodeGenOptions *CodeGenOpts = 0);
 
+  /// Create a virtual file system and replace any existing one with it.
+  /// The default is to use the real file system.
+  void createVirtualFileSystem();
+
   /// Create the file manager and replace any existing one with it.
   void createFileManager();
 

clang/lib/Basic/CMakeLists.txt

@@ -23,6 +23,7 @@ add_clang_library(clangBasic
   TokenKinds.cpp
   Version.cpp
   VersionTuple.cpp
+  VirtualFileSystem.cpp
   )
 
 # Determine Subversion revision.

clang/lib/Basic/FileManager.cpp

@@ -30,19 +30,6 @@
 #include <set>
 #include <string>
 
-// FIXME: This is terrible, we need this for ::close.
-#if !defined(_MSC_VER) && !defined(__MINGW32__)
-#include <unistd.h>
-#include <sys/uio.h>
-#else
-#include <io.h>
-#ifndef S_ISFIFO
-#define S_ISFIFO(x) (0)
-#endif
-#endif
-#if defined(LLVM_ON_UNIX)
-#include <limits.h>
-#endif
 using namespace clang;
 
 // FIXME: Enhance libsystem to support inode and other fields.
@@ -57,12 +44,6 @@ using namespace clang;
 #define NON_EXISTENT_FILE reinterpret_cast<FileEntry*>((intptr_t)-1)
 
 
-FileEntry::~FileEntry() {
-  // If this FileEntry owns an open file descriptor that never got used, close
-  // it.
-  if (FD != -1) ::close(FD);
-}
-
 class FileManager::UniqueDirContainer {
   /// UniqueDirs - Cache from ID's to existing directories/files.
   std::map<llvm::sys::fs::UniqueID, DirectoryEntry> UniqueDirs;
@@ -101,13 +82,19 @@ public:
 // Common logic.
 //===----------------------------------------------------------------------===//
 
-FileManager::FileManager(const FileSystemOptions &FSO)
+FileManager::FileManager(const FileSystemOptions &FSO,
-  : FileSystemOpts(FSO),
+                         IntrusiveRefCntPtr<vfs::FileSystem> FS)
+  : FS(FS), FileSystemOpts(FSO),
     UniqueRealDirs(*new UniqueDirContainer()),
     UniqueRealFiles(*new UniqueFileContainer()),
     SeenDirEntries(64), SeenFileEntries(64), NextFileUID(0) {
   NumDirLookups = NumFileLookups = 0;
   NumDirCacheMisses = NumFileCacheMisses = 0;
+
+  // If the caller doesn't provide a virtual file system, just grab the real
+  // file system.
+  if (!FS)
+    this->FS = vfs::getRealFileSystem();
 }
 
 FileManager::~FileManager() {
@@ -309,10 +296,9 @@ const FileEntry *FileManager::getFile(StringRef Filename, bool openFile,
   // FIXME: This will reduce the # syscalls.
 
   // Nope, there isn't.  Check to see if the file exists.
-  int FileDescriptor = -1;
+  vfs::File *F = 0;
   FileData Data;
-  if (getStatValue(InterndFileName, Data, true,
+  if (getStatValue(InterndFileName, Data, true, openFile ? &F : 0)) {
-                   openFile ? &FileDescriptor : 0)) {
     // There's no real file at the given path.
     if (!CacheFailure)
       SeenFileEntries.erase(Filename);
@@ -320,10 +306,7 @@ const FileEntry *FileManager::getFile(StringRef Filename, bool openFile,
     return 0;
   }
 
-  if (FileDescriptor != -1 && !openFile) {
+  assert(openFile || !F && "undesired open file");
-    close(FileDescriptor);
-    FileDescriptor = -1;
-  }
 
   // It exists.  See if we have already opened a file with the same inode.
   // This occurs when one dir is symlinked to another, for example.
@@ -333,8 +316,8 @@ const FileEntry *FileManager::getFile(StringRef Filename, bool openFile,
   NamedFileEnt.setValue(&UFE);
   if (UFE.getName()) { // Already have an entry with this inode, return it.
     // If the stat process opened the file, close it to avoid a FD leak.
-    if (FileDescriptor != -1)
+    if (F)
-      close(FileDescriptor);
+      delete F;
 
     return &UFE;
   }
@@ -347,7 +330,7 @@ const FileEntry *FileManager::getFile(StringRef Filename, bool openFile,
   UFE.ModTime = Data.ModTime;
   UFE.Dir     = DirInfo;
   UFE.UID     = NextFileUID++;
-  UFE.FD      = FileDescriptor;
+  UFE.File.reset(F);
   return &UFE;
 }
 
@@ -393,10 +376,8 @@ FileManager::getVirtualFile(StringRef Filename, off_t Size,
     // If we had already opened this file, close it now so we don't
     // leak the descriptor. We're not going to use the file
     // descriptor anyway, since this is a virtual file.
-    if (UFE->FD != -1) {
+    if (UFE->File)
-      close(UFE->FD);
+      UFE->closeFile();
-      UFE->FD = -1;
-    }
 
     // If we already have an entry with this inode, return it.
     if (UFE->getName())
@@ -414,7 +395,7 @@ FileManager::getVirtualFile(StringRef Filename, off_t Size,
   UFE->ModTime = ModificationTime;
   UFE->Dir     = DirInfo;
   UFE->UID     = NextFileUID++;
-  UFE->FD      = -1;
+  UFE->File.reset();
   return UFE;
 }
 
@@ -444,20 +425,18 @@ getBufferForFile(const FileEntry *Entry, std::string *ErrorStr,
 
   const char *Filename = Entry->getName();
   // If the file is already open, use the open file descriptor.
-  if (Entry->FD != -1) {
+  if (Entry->File) {
-    ec = llvm::MemoryBuffer::getOpenFile(Entry->FD, Filename, Result, FileSize);
+    ec = Entry->File->getBuffer(Filename, Result, FileSize);
     if (ErrorStr)
       *ErrorStr = ec.message();
-
+    Entry->closeFile();
-    close(Entry->FD);
-    Entry->FD = -1;
     return Result.take();
   }
 
   // Otherwise, open the file.
 
   if (FileSystemOpts.WorkingDir.empty()) {
-    ec = llvm::MemoryBuffer::getFile(Filename, Result, FileSize);
+    ec = FS->getBufferForFile(Filename, Result, FileSize);
     if (ec && ErrorStr)
       *ErrorStr = ec.message();
     return Result.take();
@@ -465,7 +444,7 @@ getBufferForFile(const FileEntry *Entry, std::string *ErrorStr,
 
   SmallString<128> FilePath(Entry->getName());
   FixupRelativePath(FilePath);
-  ec = llvm::MemoryBuffer::getFile(FilePath.str(), Result, FileSize);
+  ec = FS->getBufferForFile(FilePath.str(), Result, FileSize);
   if (ec && ErrorStr)
     *ErrorStr = ec.message();
   return Result.take();
@@ -476,7 +455,7 @@ getBufferForFile(StringRef Filename, std::string *ErrorStr) {
   OwningPtr<llvm::MemoryBuffer> Result;
   llvm::error_code ec;
   if (FileSystemOpts.WorkingDir.empty()) {
-    ec = llvm::MemoryBuffer::getFile(Filename, Result);
+    ec = FS->getBufferForFile(Filename, Result);
     if (ec && ErrorStr)
       *ErrorStr = ec.message();
     return Result.take();
@@ -484,7 +463,7 @@ getBufferForFile(StringRef Filename, std::string *ErrorStr) {
 
   SmallString<128> FilePath(Filename);
   FixupRelativePath(FilePath);
-  ec = llvm::MemoryBuffer::getFile(FilePath.c_str(), Result);
+  ec = FS->getBufferForFile(FilePath.c_str(), Result);
   if (ec && ErrorStr)
     *ErrorStr = ec.message();
   return Result.take();
@@ -496,26 +475,29 @@ getBufferForFile(StringRef Filename, std::string *ErrorStr) {
 /// false if it's an existent real file.  If FileDescriptor is NULL,
 /// do directory look-up instead of file look-up.
 bool FileManager::getStatValue(const char *Path, FileData &Data, bool isFile,
-                               int *FileDescriptor) {
+                               vfs::File **F) {
   // FIXME: FileSystemOpts shouldn't be passed in here, all paths should be
   // absolute!
   if (FileSystemOpts.WorkingDir.empty())
-    return FileSystemStatCache::get(Path, Data, isFile, FileDescriptor,
+    return FileSystemStatCache::get(Path, Data, isFile, F,StatCache.get(), *FS);
-                                    StatCache.get());
 
   SmallString<128> FilePath(Path);
   FixupRelativePath(FilePath);
 
-  return FileSystemStatCache::get(FilePath.c_str(), Data, isFile,
+  return FileSystemStatCache::get(FilePath.c_str(), Data, isFile, F,
-                                  FileDescriptor, StatCache.get());
+                                  StatCache.get(), *FS);
 }
 
 bool FileManager::getNoncachedStatValue(StringRef Path,
-                                        llvm::sys::fs::file_status &Result) {
+                                        vfs::Status &Result) {
   SmallString<128> FilePath(Path);
   FixupRelativePath(FilePath);
 
-  return llvm::sys::fs::status(FilePath.c_str(), Result);
+  llvm::ErrorOr<vfs::Status> S = FS->status(FilePath.c_str());
+  if (!S)
+    return true;
+  Result = *S;
+  return false;
 }
 
 void FileManager::invalidateCache(const FileEntry *Entry) {

clang/lib/Basic/FileSystemStatCache.cpp

@@ -12,7 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Basic/FileSystemStatCache.h"
-#include "llvm/Support/FileSystem.h"
+#include "clang/Basic/VirtualFileSystem.h"
 #include "llvm/Support/Path.h"
 
 // FIXME: This is terrible, we need this for ::close.
@@ -30,13 +30,13 @@ using namespace clang;
 
 void FileSystemStatCache::anchor() { }
 
-static void copyStatusToFileData(const llvm::sys::fs::file_status &Status,
+static void copyStatusToFileData(const vfs::Status &Status,
                                  FileData &Data) {
   Data.Size = Status.getSize();
   Data.ModTime = Status.getLastModificationTime().toEpochTime();
   Data.UniqueID = Status.getUniqueID();
-  Data.IsDirectory = is_directory(Status);
+  Data.IsDirectory = Status.isDirectory();
-  Data.IsNamedPipe = Status.type() == llvm::sys::fs::file_type::fifo_file;
+  Data.IsNamedPipe = Status.getType() == llvm::sys::fs::file_type::fifo_file;
   Data.InPCH = false;
 }
 
@@ -50,22 +50,23 @@ static void copyStatusToFileData(const llvm::sys::fs::file_status &Status,
 /// implementation can optionally fill in FileDescriptor with a valid
 /// descriptor and the client guarantees that it will close it.
 bool FileSystemStatCache::get(const char *Path, FileData &Data, bool isFile,
-                              int *FileDescriptor, FileSystemStatCache *Cache) {
+                              vfs::File **F, FileSystemStatCache *Cache,
+                              vfs::FileSystem &FS) {
   LookupResult R;
   bool isForDir = !isFile;
 
   // If we have a cache, use it to resolve the stat query.
   if (Cache)
-    R = Cache->getStat(Path, Data, isFile, FileDescriptor);
+    R = Cache->getStat(Path, Data, isFile, F, FS);
-  else if (isForDir || !FileDescriptor) {
+  else if (isForDir || !F) {
     // If this is a directory or a file descriptor is not needed and we have
     // no cache, just go to the file system.
-    llvm::sys::fs::file_status Status;
+    llvm::ErrorOr<vfs::Status> Status = FS.status(Path);
-    if (llvm::sys::fs::status(Path, Status)) {
+    if (!Status) {
       R = CacheMissing;
     } else {
       R = CacheExists;
-      copyStatusToFileData(Status, Data);
+      copyStatusToFileData(*Status, Data);
     }
   } else {
     // Otherwise, we have to go to the filesystem.  We can always just use
@@ -75,7 +76,8 @@ bool FileSystemStatCache::get(const char *Path, FileData &Data, bool isFile,
     //
     // Because of this, check to see if the file exists with 'open'.  If the
     // open succeeds, use fstat to get the stat info.
-    llvm::error_code EC = llvm::sys::fs::openFileForRead(Path, *FileDescriptor);
+    llvm::OwningPtr<vfs::File> OwnedFile;
+    llvm::error_code EC = FS.openFileForRead(Path, OwnedFile);
 
     if (EC) {
       // If the open fails, our "stat" fails.
@@ -84,16 +86,16 @@ bool FileSystemStatCache::get(const char *Path, FileData &Data, bool isFile,
       // Otherwise, the open succeeded.  Do an fstat to get the information
       // about the file.  We'll end up returning the open file descriptor to the
       // client to do what they please with it.
-      llvm::sys::fs::file_status Status;
+      llvm::ErrorOr<vfs::Status> Status = OwnedFile->status();
-      if (!llvm::sys::fs::status(*FileDescriptor, Status)) {
+      if (Status) {
         R = CacheExists;
-        copyStatusToFileData(Status, Data);
+        copyStatusToFileData(*Status, Data);
+        *F = OwnedFile.take();
       } else {
         // fstat rarely fails.  If it does, claim the initial open didn't
         // succeed.
         R = CacheMissing;
-        ::close(*FileDescriptor);
+        *F = 0;
-        *FileDescriptor = -1;
       }
     }
   }
@@ -105,9 +107,9 @@ bool FileSystemStatCache::get(const char *Path, FileData &Data, bool isFile,
   // demands.
   if (Data.IsDirectory != isForDir) {
     // If not, close the file if opened.
-    if (FileDescriptor && *FileDescriptor != -1) {
+    if (F && *F) {
-      ::close(*FileDescriptor);
+      (*F)->close();
-      *FileDescriptor = -1;
+      *F = 0;
     }
     
     return true;
@@ -118,8 +120,8 @@ bool FileSystemStatCache::get(const char *Path, FileData &Data, bool isFile,
 
 MemorizeStatCalls::LookupResult
 MemorizeStatCalls::getStat(const char *Path, FileData &Data, bool isFile,
-                           int *FileDescriptor) {
+                           vfs::File **F, vfs::FileSystem &FS) {
-  LookupResult Result = statChained(Path, Data, isFile, FileDescriptor);
+  LookupResult Result = statChained(Path, Data, isFile, F, FS);
 
   // Do not cache failed stats, it is easy to construct common inconsistent
   // situations if we do, and they are not important for PCH performance (which

clang/lib/Basic/VirtualFileSystem.cpp

@@ -0,0 +1,162 @@
+//===- VirtualFileSystem.cpp - Virtual File System Layer --------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// This file implements the VirtualFileSystem interface.
+//===----------------------------------------------------------------------===//
+
+#include "clang/Basic/VirtualFileSystem.h"
+#include "llvm/ADT/OwningPtr.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/Path.h"
+
+using namespace clang;
+using namespace clang::vfs;
+using namespace llvm;
+using llvm::sys::fs::file_status;
+using llvm::sys::fs::file_type;
+using llvm::sys::fs::perms;
+using llvm::sys::fs::UniqueID;
+
+Status::Status(const file_status &Status)
+    : UID(Status.getUniqueID()), MTime(Status.getLastModificationTime()),
+      User(Status.getUser()), Group(Status.getGroup()), Size(Status.getSize()),
+      Type(Status.type()), Perms(Status.permissions()) {}
+
+Status::Status(StringRef Name, StringRef ExternalName,
+               UniqueID UID, sys::TimeValue MTime,
+               uint32_t User, uint32_t Group, uint64_t Size,
+               file_type Type, perms Perms)
+    : Name(Name), ExternalName(ExternalName), UID(UID), MTime(MTime),
+      User(User), Group(Group), Size(Size), Type(Type), Perms(Perms) {}
+
+bool Status::equivalent(const Status &Other) const {
+  return getUniqueID() == Other.getUniqueID();
+}
+bool Status::isDirectory() const {
+  return Type == file_type::directory_file;
+}
+bool Status::isRegularFile() const {
+  return Type == file_type::regular_file;
+}
+bool Status::isOther() const {
+  return Type == exists() && !isRegularFile() && !isDirectory() && !isSymlink();
+}
+bool Status::isSymlink() const {
+  return Type == file_type::symlink_file;
+}
+bool Status::isStatusKnown() const {
+  return Type != file_type::status_error;
+}
+bool Status::exists() const {
+  return isStatusKnown() && Type != file_type::file_not_found;
+}
+
+File::~File() {}
+
+FileSystem::~FileSystem() {}
+
+error_code FileSystem::getBufferForFile(const llvm::Twine &Name,
+                                        OwningPtr<MemoryBuffer> &Result,
+                                        int64_t FileSize,
+                                        bool RequiresNullTerminator) {
+  llvm::OwningPtr<File> F;
+  if (error_code EC = openFileForRead(Name, F))
+    return EC;
+
+  error_code EC = F->getBuffer(Name, Result, FileSize, RequiresNullTerminator);
+  return EC;
+}
+
+//===-----------------------------------------------------------------------===/
+// RealFileSystem implementation
+//===-----------------------------------------------------------------------===/
+
+/// \brief Wrapper around a raw file descriptor.
+class RealFile : public File {
+  int FD;
+  friend class RealFileSystem;
+  RealFile(int FD) : FD(FD) {
+    assert(FD >= 0 && "Invalid or inactive file descriptor");
+  }
+public:
+  ~RealFile();
+  ErrorOr<Status> status() LLVM_OVERRIDE;
+  error_code getBuffer(const Twine &Name, OwningPtr<MemoryBuffer> &Result,
+                       int64_t FileSize = -1,
+                       bool RequiresNullTerminator = true) LLVM_OVERRIDE;
+  error_code close() LLVM_OVERRIDE;
+};
+RealFile::~RealFile() {
+  close();
+}
+
+ErrorOr<Status> RealFile::status() {
+  assert(FD != -1 && "cannot stat closed file");
+  file_status RealStatus;
+  if (error_code EC = sys::fs::status(FD, RealStatus))
+    return EC;
+  return Status(RealStatus);
+}
+
+error_code RealFile::getBuffer(const Twine &Name,
+                               OwningPtr<MemoryBuffer> &Result,
+                               int64_t FileSize, bool RequiresNullTerminator) {
+  assert(FD != -1 && "cannot get buffer for closed file");
+  return MemoryBuffer::getOpenFile(FD, Name.str().c_str(), Result, FileSize,
+                                   RequiresNullTerminator);
+}
+
+// FIXME: This is terrible, we need this for ::close.
+#if !defined(_MSC_VER) && !defined(__MINGW32__)
+#include <unistd.h>
+#include <sys/uio.h>
+#else
+#include <io.h>
+#ifndef S_ISFIFO
+#define S_ISFIFO(x) (0)
+#endif
+#endif
+error_code RealFile::close() {
+  if (::close(FD))
+    return error_code(errno, system_category());
+  FD = -1;
+  return error_code::success();
+}
+
+/// \brief The file system according to your operating system.
+class RealFileSystem : public FileSystem {
+public:
+  ErrorOr<Status> status(const Twine &Path) LLVM_OVERRIDE;
+  error_code openFileForRead(const Twine &Path, OwningPtr<File> &Result)
+    LLVM_OVERRIDE;
+};
+
+ErrorOr<Status> RealFileSystem::status(const Twine &Path) {
+  sys::fs::file_status RealStatus;
+  if (error_code EC = sys::fs::status(Path, RealStatus))
+    return EC;
+  Status Result(RealStatus);
+  Result.setName(Path.str());
+  Result.setExternalName(Path.str());
+  return Result;
+}
+
+error_code RealFileSystem::openFileForRead(const Twine &Name,
+                                           OwningPtr<File> &Result) {
+  int FD;
+  if (error_code EC = sys::fs::openFileForRead(Name, FD))
+    return EC;
+  Result.reset(new RealFile(FD));
+  return error_code::success();
+}
+
+IntrusiveRefCntPtr<FileSystem> vfs::getRealFileSystem() {
+  static IntrusiveRefCntPtr<FileSystem> FS = new RealFileSystem();
+  return FS;
+}

clang/lib/Frontend/ASTUnit.cpp

@@ -20,6 +20,7 @@
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Basic/TargetOptions.h"
+#include "clang/Basic/VirtualFileSystem.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/FrontendActions.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
@@ -37,7 +38,6 @@
 #include "llvm/ADT/StringSet.h"
 #include "llvm/Support/Atomic.h"
 #include "llvm/Support/CrashRecoveryContext.h"
-#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Host.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Mutex.h"
@@ -1421,7 +1421,7 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
              REnd = PreprocessorOpts.remapped_file_end();
            !AnyFileChanged && R != REnd;
            ++R) {
-        llvm::sys::fs::file_status Status;
+        vfs::Status Status;
         if (FileMgr->getNoncachedStatValue(R->second, Status)) {
           // If we can't stat the file we're remapping to, assume that something
           // horrible happened.
@@ -1457,7 +1457,7 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
         }
         
         // The file was not remapped; check whether it has changed on disk.
-        llvm::sys::fs::file_status Status;
+        vfs::Status Status;
         if (FileMgr->getNoncachedStatValue(F->first(), Status)) {
           // If we can't stat the file, assume that something horrible happened.
           AnyFileChanged = true;

clang/lib/Frontend/CacheTokens.cpp

@@ -516,8 +516,8 @@ public:
   ~StatListener() {}
 
   LookupResult getStat(const char *Path, FileData &Data, bool isFile,
-                       int *FileDescriptor) {
+                       vfs::File **F, vfs::FileSystem &FS) {
-    LookupResult Result = statChained(Path, Data, isFile, FileDescriptor);
+    LookupResult Result = statChained(Path, Data, isFile, F, FS);
 
     if (Result == CacheMissing) // Failed 'stat'.
       PM.insert(PTHEntryKeyVariant(Path), PTHEntry());

clang/lib/Frontend/ChainedIncludesSource.cpp

@@ -101,6 +101,7 @@ ChainedIncludesSource *ChainedIncludesSource::create(CompilerInstance &CI) {
     Clang->setDiagnostics(Diags.getPtr());
     Clang->setTarget(TargetInfo::CreateTargetInfo(Clang->getDiagnostics(),
                                                   &Clang->getTargetOpts()));
+    Clang->createVirtualFileSystem();
     Clang->createFileManager();
     Clang->createSourceManager(Clang->getFileManager());
     Clang->createPreprocessor();

clang/lib/Frontend/CompilerInstance.cpp

@@ -194,9 +194,14 @@ CompilerInstance::createDiagnostics(DiagnosticOptions *Opts,
   return Diags;
 }
 
+void CompilerInstance::createVirtualFileSystem() {
+  VirtualFileSystem = vfs::getRealFileSystem();
+}
+
 // File Manager
 
 void CompilerInstance::createFileManager() {
+  assert(hasVirtualFileSystem() && "expected virtual file system");
   FileMgr = new FileManager(getFileSystemOpts());
 }
 
@@ -867,6 +872,8 @@ static void compileModule(CompilerInstance &ImportingInstance,
                                    ImportingInstance.getDiagnosticClient()),
                              /*ShouldOwnClient=*/true);
 
+  Instance.setVirtualFileSystem(&ImportingInstance.getVirtualFileSystem());
+
   // Note that this module is part of the module build stack, so that we
   // can detect cycles in the module graph.
   Instance.createFileManager(); // FIXME: Adopt file manager from importer?

clang/lib/Frontend/FrontendAction.cpp

@@ -159,7 +159,6 @@ ASTConsumer* FrontendAction::CreateWrappedASTConsumer(CompilerInstance &CI,
   return new MultiplexConsumer(Consumers);
 }
 
-
 bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
                                      const FrontendInputFile &Input) {
   assert(!Instance && "Already processing a source file!");
@@ -213,6 +212,8 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
   }
 
   // Set up the file and source managers, if needed.
+  if (!CI.hasVirtualFileSystem())
+    CI.createVirtualFileSystem();
   if (!CI.hasFileManager())
     CI.createFileManager();
   if (!CI.hasSourceManager())

clang/lib/Lex/PTHLexer.cpp

@@ -675,13 +675,13 @@ public:
   ~PTHStatCache() {}
 
   LookupResult getStat(const char *Path, FileData &Data, bool isFile,
-                       int *FileDescriptor) {
+                       vfs::File **F, vfs::FileSystem &FS) {
     // Do the lookup for the file's data in the PTH file.
     CacheTy::iterator I = Cache.find(Path);
 
     // If we don't get a hit in the PTH file just forward to 'stat'.
     if (I == Cache.end())
-      return statChained(Path, Data, isFile, FileDescriptor);
+      return statChained(Path, Data, isFile, F, FS);
 
     const PTHStatData &D = *I;
 

clang/lib/Serialization/ModuleManager.cpp

@@ -89,7 +89,7 @@ ModuleManager::addModule(StringRef FileName, ModuleKind Type,
     New->InputFilesValidationTimestamp = 0;
     if (New->Kind == MK_Module) {
       std::string TimestampFilename = New->getTimestampFilename();
-      llvm::sys::fs::file_status Status;
+      vfs::Status Status;
       // A cached stat value would be fine as well.
       if (!FileMgr.getNoncachedStatValue(TimestampFilename, Status))
         New->InputFilesValidationTimestamp =

clang/unittests/Basic/FileManagerTest.cpp

@@ -48,7 +48,7 @@ public:
 
   // Implement FileSystemStatCache::getStat().
   virtual LookupResult getStat(const char *Path, FileData &Data, bool isFile,
-                               int *FileDescriptor) {
+                               vfs::File **F, vfs::FileSystem &FS) {
     if (StatCalls.count(Path) != 0) {
       Data = StatCalls[Path];
       return CacheExists;