From 090610d37a6bc50dabac779f97e158154cbe9bd7 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Wed, 19 Feb 2014 00:10:30 +0000 Subject: [PATCH] Initial implementation of virtual file system This adds the minimum virtual file system support to start migrating FileManager onto the VFS. Originally discussed here: http://lists.cs.uiuc.edu/pipermail/cfe-dev/2014-February/035188.html Differential Revision: http://llvm-reviews.chandlerc.com/D2745 llvm-svn: 201618 --- clang/include/clang/Basic/FileManager.h | 34 ++-- .../include/clang/Basic/FileSystemStatCache.h | 23 ++- clang/include/clang/Basic/VirtualFileSystem.h | 127 ++++++++++++++ .../include/clang/Frontend/CompilerInstance.h | 24 +++ clang/lib/Basic/CMakeLists.txt | 1 + clang/lib/Basic/FileManager.cpp | 86 ++++------ clang/lib/Basic/FileSystemStatCache.cpp | 44 ++--- clang/lib/Basic/VirtualFileSystem.cpp | 162 ++++++++++++++++++ clang/lib/Frontend/ASTUnit.cpp | 6 +- clang/lib/Frontend/CacheTokens.cpp | 4 +- clang/lib/Frontend/ChainedIncludesSource.cpp | 1 + clang/lib/Frontend/CompilerInstance.cpp | 7 + clang/lib/Frontend/FrontendAction.cpp | 3 +- clang/lib/Lex/PTHLexer.cpp | 4 +- clang/lib/Serialization/ModuleManager.cpp | 2 +- clang/unittests/Basic/FileManagerTest.cpp | 2 +- 16 files changed, 423 insertions(+), 107 deletions(-) create mode 100644 clang/include/clang/Basic/VirtualFileSystem.h create mode 100644 clang/lib/Basic/VirtualFileSystem.cpp diff --git a/clang/include/clang/Basic/FileManager.h b/clang/include/clang/Basic/FileManager.h index 5f2248185561..a4af28037951 100644 --- a/clang/include/clang/Basic/FileManager.h +++ b/clang/include/clang/Basic/FileManager.h @@ -17,6 +17,7 @@ #include "clang/Basic/FileSystemOptions.h" #include "clang/Basic/LLVM.h" +#include "clang/Basic/VirtualFileSystem.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/OwningPtr.h" @@ -24,7 +25,6 @@ #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/Allocator.h" -#include "llvm/Support/FileSystem.h" // FIXME: Enhance libsystem to support inode and other fields in stat. #include @@ -55,7 +55,7 @@ public: /// \brief Cached information about one file (either on disk /// or in the virtual file system). /// -/// If the 'FD' member is valid, then this FileEntry has an open file +/// If the 'File' member is valid, then this FileEntry has an open file /// descriptor for the file. class FileEntry { const char *Name; // Name of the file. @@ -67,31 +67,33 @@ class FileEntry { bool IsNamedPipe; bool InPCH; - /// FD - The file descriptor for the file entry if it is opened and owned - /// by the FileEntry. If not, this is set to -1. - mutable int FD; + /// \brief The open file, if it is owned by the \p FileEntry. + mutable OwningPtr File; friend class FileManager; + void closeFile() const { + File.reset(0); // rely on destructor to close File + } + public: FileEntry(llvm::sys::fs::UniqueID UniqueID, bool IsNamedPipe, bool InPCH) - : Name(0), UniqueID(UniqueID), IsNamedPipe(IsNamedPipe), InPCH(InPCH), - FD(-1) {} + : Name(0), UniqueID(UniqueID), IsNamedPipe(IsNamedPipe), InPCH(InPCH) + {} // Add a default constructor for use with llvm::StringMap FileEntry() - : Name(0), UniqueID(0, 0), IsNamedPipe(false), InPCH(false), FD(-1) {} + : Name(0), UniqueID(0, 0), IsNamedPipe(false), InPCH(false) + {} FileEntry(const FileEntry &FE) { memcpy(this, &FE, sizeof(FE)); - assert(FD == -1 && "Cannot copy a file-owning FileEntry"); + assert(!File && "Cannot copy a file-owning FileEntry"); } void operator=(const FileEntry &FE) { memcpy(this, &FE, sizeof(FE)); - assert(FD == -1 && "Cannot assign a file-owning FileEntry"); + assert(!File && "Cannot assign a file-owning FileEntry"); } - ~FileEntry(); - const char *getName() const { return Name; } off_t getSize() const { return Size; } unsigned getUID() const { return UID; } @@ -119,6 +121,7 @@ struct FileData; /// as a single file. /// class FileManager : public RefCountedBase { + IntrusiveRefCntPtr FS; FileSystemOptions FileSystemOpts; class UniqueDirContainer; @@ -172,14 +175,15 @@ class FileManager : public RefCountedBase { OwningPtr StatCache; bool getStatValue(const char *Path, FileData &Data, bool isFile, - int *FileDescriptor); + vfs::File **F); /// Add all ancestors of the given path (pointing to either a file /// or a directory) as virtual directories. void addAncestorsAsVirtualDirs(StringRef Path); public: - FileManager(const FileSystemOptions &FileSystemOpts); + FileManager(const FileSystemOptions &FileSystemOpts, + llvm::IntrusiveRefCntPtr FS = 0); ~FileManager(); /// \brief Installs the provided FileSystemStatCache object within @@ -248,7 +252,7 @@ public: /// /// \returns false on success, true on error. bool getNoncachedStatValue(StringRef Path, - llvm::sys::fs::file_status &Result); + vfs::Status &Result); /// \brief Remove the real file \p Entry from the cache. void invalidateCache(const FileEntry *Entry); diff --git a/clang/include/clang/Basic/FileSystemStatCache.h b/clang/include/clang/Basic/FileSystemStatCache.h index 23d82569437f..913e41e4ca8b 100644 --- a/clang/include/clang/Basic/FileSystemStatCache.h +++ b/clang/include/clang/Basic/FileSystemStatCache.h @@ -24,6 +24,11 @@ namespace clang { +namespace vfs { +class File; +class FileSystem; +} + struct FileData { uint64_t Size; time_t ModTime; @@ -57,10 +62,11 @@ public: /// If isFile is true, then this lookup should only return success for files /// (not directories). If it is false this lookup should only return /// success for directories (not files). On a successful file lookup, the - /// implementation can optionally fill in FileDescriptor with a valid - /// descriptor and the client guarantees that it will close it. + /// implementation can optionally fill in \p F with a valid \p File object and + /// the client guarantees that it will close it. static bool get(const char *Path, FileData &Data, bool isFile, - int *FileDescriptor, FileSystemStatCache *Cache); + vfs::File **F, FileSystemStatCache *Cache, + vfs::FileSystem &FS); /// \brief Sets the next stat call cache in the chain of stat caches. /// Takes ownership of the given stat cache. @@ -78,17 +84,16 @@ public: protected: virtual LookupResult getStat(const char *Path, FileData &Data, bool isFile, - int *FileDescriptor) = 0; + vfs::File **F, vfs::FileSystem &FS) = 0; LookupResult statChained(const char *Path, FileData &Data, bool isFile, - int *FileDescriptor) { + vfs::File **F, vfs::FileSystem &FS) { if (FileSystemStatCache *Next = getNextStatCache()) - return Next->getStat(Path, Data, isFile, FileDescriptor); + return Next->getStat(Path, Data, isFile, F, FS); // If we hit the end of the list of stat caches to try, just compute and // return it without a cache. - return get(Path, Data, isFile, FileDescriptor, 0) ? CacheMissing - : CacheExists; + return get(Path, Data, isFile, F, 0, FS) ? CacheMissing : CacheExists; } }; @@ -107,7 +112,7 @@ public: iterator end() const { return StatCalls.end(); } virtual LookupResult getStat(const char *Path, FileData &Data, bool isFile, - int *FileDescriptor); + vfs::File **F, vfs::FileSystem &FS); }; } // end namespace clang diff --git a/clang/include/clang/Basic/VirtualFileSystem.h b/clang/include/clang/Basic/VirtualFileSystem.h new file mode 100644 index 000000000000..9bd6b2bc63b6 --- /dev/null +++ b/clang/include/clang/Basic/VirtualFileSystem.h @@ -0,0 +1,127 @@ +//===- VirtualFileSystem.h - Virtual File System Layer ----------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// \brief Defines the virtual file system interface vfs::FileSystem. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_BASIC_VIRTUAL_FILE_SYSTEM_H +#define LLVM_CLANG_BASIC_VIRTUAL_FILE_SYSTEM_H + +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/ErrorOr.h" + +namespace llvm { +template class OwningPtr; +class MemoryBuffer; +} + +namespace clang { +namespace vfs { + +/// \brief The result of a \p status operation. +class Status { + std::string Name; + std::string ExternalName; + llvm::sys::fs::UniqueID UID; + llvm::sys::TimeValue MTime; + uint32_t User; + uint32_t Group; + uint64_t Size; + llvm::sys::fs::file_type Type; + llvm::sys::fs::perms Perms; + +public: + Status() : Type(llvm::sys::fs::file_type::status_error) {} + Status(const llvm::sys::fs::file_status &Status); + Status(llvm::StringRef Name, llvm::StringRef RealName, + llvm::sys::fs::UniqueID UID, llvm::sys::TimeValue MTime, + uint32_t User, uint32_t Group, uint64_t Size, + llvm::sys::fs::file_type Type, llvm::sys::fs::perms Perms); + + /// \brief Returns the name this status was looked up by. + llvm::StringRef getName() const { return Name; } + + /// \brief Returns the name to use outside the compiler. + /// + /// For example, in diagnostics or debug info we should use this name. + llvm::StringRef getExternalName() const { return ExternalName; } + + void setName(llvm::StringRef N) { Name = N; } + void setExternalName(llvm::StringRef N) { ExternalName = N; } + + /// @name Status interface from llvm::sys::fs + /// @{ + llvm::sys::fs::file_type getType() const { return Type; } + llvm::sys::fs::perms getPermissions() const { return Perms; } + llvm::sys::TimeValue getLastModificationTime() const { return MTime; } + llvm::sys::fs::UniqueID getUniqueID() const { return UID; } + uint32_t getUser() const { return User; } + uint32_t getGroup() const { return Group; } + uint64_t getSize() const { return Size; } + void setType(llvm::sys::fs::file_type v) { Type = v; } + void setPermissions(llvm::sys::fs::perms p) { Perms = p; } + /// @} + /// @name Status queries + /// These are static queries in llvm::sys::fs. + /// @{ + bool equivalent(const Status &Other) const; + bool isDirectory() const; + bool isRegularFile() const; + bool isOther() const; + bool isSymlink() const; + bool isStatusKnown() const; + bool exists() const; + /// @} +}; + +/// \brief Represents an open file. +class File { +public: + /// \brief Destroy the file after closing it (if open). + /// Sub-classes should generally call close() inside their destructors. We + /// cannot do that from the base class, since close is virtual. + virtual ~File(); + /// \brief Get the status of the file. + virtual llvm::ErrorOr status() = 0; + /// \brief Get the contents of the file as a \p MemoryBuffer. + virtual llvm::error_code + getBuffer(const llvm::Twine &Name, + llvm::OwningPtr &Result, int64_t FileSize = -1, + bool RequiresNullTerminator = true) = 0; + /// \brief Closes the file. + virtual llvm::error_code close() = 0; +}; + +/// \brief The virtual file system interface. +class FileSystem : public llvm::RefCountedBase { +public: + virtual ~FileSystem(); + + /// \brief Get the status of the entry at \p Path, if one exists. + virtual llvm::ErrorOr status(const llvm::Twine &Path) = 0; + /// \brief Get a \p File object for the file at \p Path, if one exists. + virtual llvm::error_code openFileForRead(const llvm::Twine &Path, + llvm::OwningPtr &Result) = 0; + + /// This is a convenience method that opens a file, gets its content and then + /// closes the file. + llvm::error_code getBufferForFile(const llvm::Twine &Name, + llvm::OwningPtr &Result, + int64_t FileSize = -1, + bool RequiresNullTerminator = true); +}; + +/// \brief Gets an \p vfs::FileSystem for the 'real' file system, as seen by +/// the operating system. +llvm::IntrusiveRefCntPtr getRealFileSystem(); + +} // end namespace vfs +} // end namespace clang +#endif // LLVM_CLANG_BASIC_VIRTUAL_FILE_SYSTEM_H diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h index fa20ae48e4f0..8cfc8ee1d299 100644 --- a/clang/include/clang/Frontend/CompilerInstance.h +++ b/clang/include/clang/Frontend/CompilerInstance.h @@ -75,6 +75,9 @@ class CompilerInstance : public ModuleLoader { /// The target being compiled for. IntrusiveRefCntPtr Target; + /// The virtual file system. + IntrusiveRefCntPtr VirtualFileSystem; + /// The file manager. IntrusiveRefCntPtr FileMgr; @@ -313,6 +316,23 @@ public: /// Replace the current diagnostics engine. void setTarget(TargetInfo *Value); + /// } + /// @name Virtual File System + /// { + + bool hasVirtualFileSystem() const { return VirtualFileSystem != 0; } + + vfs::FileSystem &getVirtualFileSystem() const { + assert(hasVirtualFileSystem() && + "Compiler instance has no virtual file system"); + return *VirtualFileSystem; + } + + /// \brief Replace the current virtual file system. + void setVirtualFileSystem(IntrusiveRefCntPtr FS) { + VirtualFileSystem = FS; + } + /// } /// @name File Manager /// { @@ -527,6 +547,10 @@ public: bool ShouldOwnClient = true, const CodeGenOptions *CodeGenOpts = 0); + /// Create a virtual file system and replace any existing one with it. + /// The default is to use the real file system. + void createVirtualFileSystem(); + /// Create the file manager and replace any existing one with it. void createFileManager(); diff --git a/clang/lib/Basic/CMakeLists.txt b/clang/lib/Basic/CMakeLists.txt index 43622eb33de1..2efe8127b9a5 100644 --- a/clang/lib/Basic/CMakeLists.txt +++ b/clang/lib/Basic/CMakeLists.txt @@ -23,6 +23,7 @@ add_clang_library(clangBasic TokenKinds.cpp Version.cpp VersionTuple.cpp + VirtualFileSystem.cpp ) # Determine Subversion revision. diff --git a/clang/lib/Basic/FileManager.cpp b/clang/lib/Basic/FileManager.cpp index af9b2663cbfa..08f19fb2006e 100644 --- a/clang/lib/Basic/FileManager.cpp +++ b/clang/lib/Basic/FileManager.cpp @@ -30,19 +30,6 @@ #include #include -// FIXME: This is terrible, we need this for ::close. -#if !defined(_MSC_VER) && !defined(__MINGW32__) -#include -#include -#else -#include -#ifndef S_ISFIFO -#define S_ISFIFO(x) (0) -#endif -#endif -#if defined(LLVM_ON_UNIX) -#include -#endif using namespace clang; // FIXME: Enhance libsystem to support inode and other fields. @@ -57,12 +44,6 @@ using namespace clang; #define NON_EXISTENT_FILE reinterpret_cast((intptr_t)-1) -FileEntry::~FileEntry() { - // If this FileEntry owns an open file descriptor that never got used, close - // it. - if (FD != -1) ::close(FD); -} - class FileManager::UniqueDirContainer { /// UniqueDirs - Cache from ID's to existing directories/files. std::map UniqueDirs; @@ -101,13 +82,19 @@ public: // Common logic. //===----------------------------------------------------------------------===// -FileManager::FileManager(const FileSystemOptions &FSO) - : FileSystemOpts(FSO), +FileManager::FileManager(const FileSystemOptions &FSO, + IntrusiveRefCntPtr FS) + : FS(FS), FileSystemOpts(FSO), UniqueRealDirs(*new UniqueDirContainer()), UniqueRealFiles(*new UniqueFileContainer()), SeenDirEntries(64), SeenFileEntries(64), NextFileUID(0) { NumDirLookups = NumFileLookups = 0; NumDirCacheMisses = NumFileCacheMisses = 0; + + // If the caller doesn't provide a virtual file system, just grab the real + // file system. + if (!FS) + this->FS = vfs::getRealFileSystem(); } FileManager::~FileManager() { @@ -309,10 +296,9 @@ const FileEntry *FileManager::getFile(StringRef Filename, bool openFile, // FIXME: This will reduce the # syscalls. // Nope, there isn't. Check to see if the file exists. - int FileDescriptor = -1; + vfs::File *F = 0; FileData Data; - if (getStatValue(InterndFileName, Data, true, - openFile ? &FileDescriptor : 0)) { + if (getStatValue(InterndFileName, Data, true, openFile ? &F : 0)) { // There's no real file at the given path. if (!CacheFailure) SeenFileEntries.erase(Filename); @@ -320,10 +306,7 @@ const FileEntry *FileManager::getFile(StringRef Filename, bool openFile, return 0; } - if (FileDescriptor != -1 && !openFile) { - close(FileDescriptor); - FileDescriptor = -1; - } + assert(openFile || !F && "undesired open file"); // It exists. See if we have already opened a file with the same inode. // This occurs when one dir is symlinked to another, for example. @@ -333,8 +316,8 @@ const FileEntry *FileManager::getFile(StringRef Filename, bool openFile, NamedFileEnt.setValue(&UFE); if (UFE.getName()) { // Already have an entry with this inode, return it. // If the stat process opened the file, close it to avoid a FD leak. - if (FileDescriptor != -1) - close(FileDescriptor); + if (F) + delete F; return &UFE; } @@ -347,7 +330,7 @@ const FileEntry *FileManager::getFile(StringRef Filename, bool openFile, UFE.ModTime = Data.ModTime; UFE.Dir = DirInfo; UFE.UID = NextFileUID++; - UFE.FD = FileDescriptor; + UFE.File.reset(F); return &UFE; } @@ -393,10 +376,8 @@ FileManager::getVirtualFile(StringRef Filename, off_t Size, // If we had already opened this file, close it now so we don't // leak the descriptor. We're not going to use the file // descriptor anyway, since this is a virtual file. - if (UFE->FD != -1) { - close(UFE->FD); - UFE->FD = -1; - } + if (UFE->File) + UFE->closeFile(); // If we already have an entry with this inode, return it. if (UFE->getName()) @@ -414,7 +395,7 @@ FileManager::getVirtualFile(StringRef Filename, off_t Size, UFE->ModTime = ModificationTime; UFE->Dir = DirInfo; UFE->UID = NextFileUID++; - UFE->FD = -1; + UFE->File.reset(); return UFE; } @@ -444,20 +425,18 @@ getBufferForFile(const FileEntry *Entry, std::string *ErrorStr, const char *Filename = Entry->getName(); // If the file is already open, use the open file descriptor. - if (Entry->FD != -1) { - ec = llvm::MemoryBuffer::getOpenFile(Entry->FD, Filename, Result, FileSize); + if (Entry->File) { + ec = Entry->File->getBuffer(Filename, Result, FileSize); if (ErrorStr) *ErrorStr = ec.message(); - - close(Entry->FD); - Entry->FD = -1; + Entry->closeFile(); return Result.take(); } // Otherwise, open the file. if (FileSystemOpts.WorkingDir.empty()) { - ec = llvm::MemoryBuffer::getFile(Filename, Result, FileSize); + ec = FS->getBufferForFile(Filename, Result, FileSize); if (ec && ErrorStr) *ErrorStr = ec.message(); return Result.take(); @@ -465,7 +444,7 @@ getBufferForFile(const FileEntry *Entry, std::string *ErrorStr, SmallString<128> FilePath(Entry->getName()); FixupRelativePath(FilePath); - ec = llvm::MemoryBuffer::getFile(FilePath.str(), Result, FileSize); + ec = FS->getBufferForFile(FilePath.str(), Result, FileSize); if (ec && ErrorStr) *ErrorStr = ec.message(); return Result.take(); @@ -476,7 +455,7 @@ getBufferForFile(StringRef Filename, std::string *ErrorStr) { OwningPtr Result; llvm::error_code ec; if (FileSystemOpts.WorkingDir.empty()) { - ec = llvm::MemoryBuffer::getFile(Filename, Result); + ec = FS->getBufferForFile(Filename, Result); if (ec && ErrorStr) *ErrorStr = ec.message(); return Result.take(); @@ -484,7 +463,7 @@ getBufferForFile(StringRef Filename, std::string *ErrorStr) { SmallString<128> FilePath(Filename); FixupRelativePath(FilePath); - ec = llvm::MemoryBuffer::getFile(FilePath.c_str(), Result); + ec = FS->getBufferForFile(FilePath.c_str(), Result); if (ec && ErrorStr) *ErrorStr = ec.message(); return Result.take(); @@ -496,26 +475,29 @@ getBufferForFile(StringRef Filename, std::string *ErrorStr) { /// false if it's an existent real file. If FileDescriptor is NULL, /// do directory look-up instead of file look-up. bool FileManager::getStatValue(const char *Path, FileData &Data, bool isFile, - int *FileDescriptor) { + vfs::File **F) { // FIXME: FileSystemOpts shouldn't be passed in here, all paths should be // absolute! if (FileSystemOpts.WorkingDir.empty()) - return FileSystemStatCache::get(Path, Data, isFile, FileDescriptor, - StatCache.get()); + return FileSystemStatCache::get(Path, Data, isFile, F,StatCache.get(), *FS); SmallString<128> FilePath(Path); FixupRelativePath(FilePath); - return FileSystemStatCache::get(FilePath.c_str(), Data, isFile, - FileDescriptor, StatCache.get()); + return FileSystemStatCache::get(FilePath.c_str(), Data, isFile, F, + StatCache.get(), *FS); } bool FileManager::getNoncachedStatValue(StringRef Path, - llvm::sys::fs::file_status &Result) { + vfs::Status &Result) { SmallString<128> FilePath(Path); FixupRelativePath(FilePath); - return llvm::sys::fs::status(FilePath.c_str(), Result); + llvm::ErrorOr S = FS->status(FilePath.c_str()); + if (!S) + return true; + Result = *S; + return false; } void FileManager::invalidateCache(const FileEntry *Entry) { diff --git a/clang/lib/Basic/FileSystemStatCache.cpp b/clang/lib/Basic/FileSystemStatCache.cpp index 7a01bffcd95f..b225facbad95 100644 --- a/clang/lib/Basic/FileSystemStatCache.cpp +++ b/clang/lib/Basic/FileSystemStatCache.cpp @@ -12,7 +12,7 @@ //===----------------------------------------------------------------------===// #include "clang/Basic/FileSystemStatCache.h" -#include "llvm/Support/FileSystem.h" +#include "clang/Basic/VirtualFileSystem.h" #include "llvm/Support/Path.h" // FIXME: This is terrible, we need this for ::close. @@ -30,13 +30,13 @@ using namespace clang; void FileSystemStatCache::anchor() { } -static void copyStatusToFileData(const llvm::sys::fs::file_status &Status, +static void copyStatusToFileData(const vfs::Status &Status, FileData &Data) { Data.Size = Status.getSize(); Data.ModTime = Status.getLastModificationTime().toEpochTime(); Data.UniqueID = Status.getUniqueID(); - Data.IsDirectory = is_directory(Status); - Data.IsNamedPipe = Status.type() == llvm::sys::fs::file_type::fifo_file; + Data.IsDirectory = Status.isDirectory(); + Data.IsNamedPipe = Status.getType() == llvm::sys::fs::file_type::fifo_file; Data.InPCH = false; } @@ -50,22 +50,23 @@ static void copyStatusToFileData(const llvm::sys::fs::file_status &Status, /// implementation can optionally fill in FileDescriptor with a valid /// descriptor and the client guarantees that it will close it. bool FileSystemStatCache::get(const char *Path, FileData &Data, bool isFile, - int *FileDescriptor, FileSystemStatCache *Cache) { + vfs::File **F, FileSystemStatCache *Cache, + vfs::FileSystem &FS) { LookupResult R; bool isForDir = !isFile; // If we have a cache, use it to resolve the stat query. if (Cache) - R = Cache->getStat(Path, Data, isFile, FileDescriptor); - else if (isForDir || !FileDescriptor) { + R = Cache->getStat(Path, Data, isFile, F, FS); + else if (isForDir || !F) { // If this is a directory or a file descriptor is not needed and we have // no cache, just go to the file system. - llvm::sys::fs::file_status Status; - if (llvm::sys::fs::status(Path, Status)) { + llvm::ErrorOr Status = FS.status(Path); + if (!Status) { R = CacheMissing; } else { R = CacheExists; - copyStatusToFileData(Status, Data); + copyStatusToFileData(*Status, Data); } } else { // Otherwise, we have to go to the filesystem. We can always just use @@ -75,7 +76,8 @@ bool FileSystemStatCache::get(const char *Path, FileData &Data, bool isFile, // // Because of this, check to see if the file exists with 'open'. If the // open succeeds, use fstat to get the stat info. - llvm::error_code EC = llvm::sys::fs::openFileForRead(Path, *FileDescriptor); + llvm::OwningPtr OwnedFile; + llvm::error_code EC = FS.openFileForRead(Path, OwnedFile); if (EC) { // If the open fails, our "stat" fails. @@ -84,16 +86,16 @@ bool FileSystemStatCache::get(const char *Path, FileData &Data, bool isFile, // Otherwise, the open succeeded. Do an fstat to get the information // about the file. We'll end up returning the open file descriptor to the // client to do what they please with it. - llvm::sys::fs::file_status Status; - if (!llvm::sys::fs::status(*FileDescriptor, Status)) { + llvm::ErrorOr Status = OwnedFile->status(); + if (Status) { R = CacheExists; - copyStatusToFileData(Status, Data); + copyStatusToFileData(*Status, Data); + *F = OwnedFile.take(); } else { // fstat rarely fails. If it does, claim the initial open didn't // succeed. R = CacheMissing; - ::close(*FileDescriptor); - *FileDescriptor = -1; + *F = 0; } } } @@ -105,9 +107,9 @@ bool FileSystemStatCache::get(const char *Path, FileData &Data, bool isFile, // demands. if (Data.IsDirectory != isForDir) { // If not, close the file if opened. - if (FileDescriptor && *FileDescriptor != -1) { - ::close(*FileDescriptor); - *FileDescriptor = -1; + if (F && *F) { + (*F)->close(); + *F = 0; } return true; @@ -118,8 +120,8 @@ bool FileSystemStatCache::get(const char *Path, FileData &Data, bool isFile, MemorizeStatCalls::LookupResult MemorizeStatCalls::getStat(const char *Path, FileData &Data, bool isFile, - int *FileDescriptor) { - LookupResult Result = statChained(Path, Data, isFile, FileDescriptor); + vfs::File **F, vfs::FileSystem &FS) { + LookupResult Result = statChained(Path, Data, isFile, F, FS); // Do not cache failed stats, it is easy to construct common inconsistent // situations if we do, and they are not important for PCH performance (which diff --git a/clang/lib/Basic/VirtualFileSystem.cpp b/clang/lib/Basic/VirtualFileSystem.cpp new file mode 100644 index 000000000000..0345ed153e3d --- /dev/null +++ b/clang/lib/Basic/VirtualFileSystem.cpp @@ -0,0 +1,162 @@ +//===- VirtualFileSystem.cpp - Virtual File System Layer --------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// This file implements the VirtualFileSystem interface. +//===----------------------------------------------------------------------===// + +#include "clang/Basic/VirtualFileSystem.h" +#include "llvm/ADT/OwningPtr.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/Path.h" + +using namespace clang; +using namespace clang::vfs; +using namespace llvm; +using llvm::sys::fs::file_status; +using llvm::sys::fs::file_type; +using llvm::sys::fs::perms; +using llvm::sys::fs::UniqueID; + +Status::Status(const file_status &Status) + : UID(Status.getUniqueID()), MTime(Status.getLastModificationTime()), + User(Status.getUser()), Group(Status.getGroup()), Size(Status.getSize()), + Type(Status.type()), Perms(Status.permissions()) {} + +Status::Status(StringRef Name, StringRef ExternalName, + UniqueID UID, sys::TimeValue MTime, + uint32_t User, uint32_t Group, uint64_t Size, + file_type Type, perms Perms) + : Name(Name), ExternalName(ExternalName), UID(UID), MTime(MTime), + User(User), Group(Group), Size(Size), Type(Type), Perms(Perms) {} + +bool Status::equivalent(const Status &Other) const { + return getUniqueID() == Other.getUniqueID(); +} +bool Status::isDirectory() const { + return Type == file_type::directory_file; +} +bool Status::isRegularFile() const { + return Type == file_type::regular_file; +} +bool Status::isOther() const { + return Type == exists() && !isRegularFile() && !isDirectory() && !isSymlink(); +} +bool Status::isSymlink() const { + return Type == file_type::symlink_file; +} +bool Status::isStatusKnown() const { + return Type != file_type::status_error; +} +bool Status::exists() const { + return isStatusKnown() && Type != file_type::file_not_found; +} + +File::~File() {} + +FileSystem::~FileSystem() {} + +error_code FileSystem::getBufferForFile(const llvm::Twine &Name, + OwningPtr &Result, + int64_t FileSize, + bool RequiresNullTerminator) { + llvm::OwningPtr F; + if (error_code EC = openFileForRead(Name, F)) + return EC; + + error_code EC = F->getBuffer(Name, Result, FileSize, RequiresNullTerminator); + return EC; +} + +//===-----------------------------------------------------------------------===/ +// RealFileSystem implementation +//===-----------------------------------------------------------------------===/ + +/// \brief Wrapper around a raw file descriptor. +class RealFile : public File { + int FD; + friend class RealFileSystem; + RealFile(int FD) : FD(FD) { + assert(FD >= 0 && "Invalid or inactive file descriptor"); + } +public: + ~RealFile(); + ErrorOr status() LLVM_OVERRIDE; + error_code getBuffer(const Twine &Name, OwningPtr &Result, + int64_t FileSize = -1, + bool RequiresNullTerminator = true) LLVM_OVERRIDE; + error_code close() LLVM_OVERRIDE; +}; +RealFile::~RealFile() { + close(); +} + +ErrorOr RealFile::status() { + assert(FD != -1 && "cannot stat closed file"); + file_status RealStatus; + if (error_code EC = sys::fs::status(FD, RealStatus)) + return EC; + return Status(RealStatus); +} + +error_code RealFile::getBuffer(const Twine &Name, + OwningPtr &Result, + int64_t FileSize, bool RequiresNullTerminator) { + assert(FD != -1 && "cannot get buffer for closed file"); + return MemoryBuffer::getOpenFile(FD, Name.str().c_str(), Result, FileSize, + RequiresNullTerminator); +} + +// FIXME: This is terrible, we need this for ::close. +#if !defined(_MSC_VER) && !defined(__MINGW32__) +#include +#include +#else +#include +#ifndef S_ISFIFO +#define S_ISFIFO(x) (0) +#endif +#endif +error_code RealFile::close() { + if (::close(FD)) + return error_code(errno, system_category()); + FD = -1; + return error_code::success(); +} + +/// \brief The file system according to your operating system. +class RealFileSystem : public FileSystem { +public: + ErrorOr status(const Twine &Path) LLVM_OVERRIDE; + error_code openFileForRead(const Twine &Path, OwningPtr &Result) + LLVM_OVERRIDE; +}; + +ErrorOr RealFileSystem::status(const Twine &Path) { + sys::fs::file_status RealStatus; + if (error_code EC = sys::fs::status(Path, RealStatus)) + return EC; + Status Result(RealStatus); + Result.setName(Path.str()); + Result.setExternalName(Path.str()); + return Result; +} + +error_code RealFileSystem::openFileForRead(const Twine &Name, + OwningPtr &Result) { + int FD; + if (error_code EC = sys::fs::openFileForRead(Name, FD)) + return EC; + Result.reset(new RealFile(FD)); + return error_code::success(); +} + +IntrusiveRefCntPtr vfs::getRealFileSystem() { + static IntrusiveRefCntPtr FS = new RealFileSystem(); + return FS; +} diff --git a/clang/lib/Frontend/ASTUnit.cpp b/clang/lib/Frontend/ASTUnit.cpp index 7f6ae5680e77..d617b368065c 100644 --- a/clang/lib/Frontend/ASTUnit.cpp +++ b/clang/lib/Frontend/ASTUnit.cpp @@ -20,6 +20,7 @@ #include "clang/Basic/Diagnostic.h" #include "clang/Basic/TargetInfo.h" #include "clang/Basic/TargetOptions.h" +#include "clang/Basic/VirtualFileSystem.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/FrontendActions.h" #include "clang/Frontend/FrontendDiagnostic.h" @@ -37,7 +38,6 @@ #include "llvm/ADT/StringSet.h" #include "llvm/Support/Atomic.h" #include "llvm/Support/CrashRecoveryContext.h" -#include "llvm/Support/FileSystem.h" #include "llvm/Support/Host.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Mutex.h" @@ -1421,7 +1421,7 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble( REnd = PreprocessorOpts.remapped_file_end(); !AnyFileChanged && R != REnd; ++R) { - llvm::sys::fs::file_status Status; + vfs::Status Status; if (FileMgr->getNoncachedStatValue(R->second, Status)) { // If we can't stat the file we're remapping to, assume that something // horrible happened. @@ -1457,7 +1457,7 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble( } // The file was not remapped; check whether it has changed on disk. - llvm::sys::fs::file_status Status; + vfs::Status Status; if (FileMgr->getNoncachedStatValue(F->first(), Status)) { // If we can't stat the file, assume that something horrible happened. AnyFileChanged = true; diff --git a/clang/lib/Frontend/CacheTokens.cpp b/clang/lib/Frontend/CacheTokens.cpp index 0c30b049579d..5cb364e3f5c9 100644 --- a/clang/lib/Frontend/CacheTokens.cpp +++ b/clang/lib/Frontend/CacheTokens.cpp @@ -516,8 +516,8 @@ public: ~StatListener() {} LookupResult getStat(const char *Path, FileData &Data, bool isFile, - int *FileDescriptor) { - LookupResult Result = statChained(Path, Data, isFile, FileDescriptor); + vfs::File **F, vfs::FileSystem &FS) { + LookupResult Result = statChained(Path, Data, isFile, F, FS); if (Result == CacheMissing) // Failed 'stat'. PM.insert(PTHEntryKeyVariant(Path), PTHEntry()); diff --git a/clang/lib/Frontend/ChainedIncludesSource.cpp b/clang/lib/Frontend/ChainedIncludesSource.cpp index 442177ec3197..d6d63bfb0050 100644 --- a/clang/lib/Frontend/ChainedIncludesSource.cpp +++ b/clang/lib/Frontend/ChainedIncludesSource.cpp @@ -101,6 +101,7 @@ ChainedIncludesSource *ChainedIncludesSource::create(CompilerInstance &CI) { Clang->setDiagnostics(Diags.getPtr()); Clang->setTarget(TargetInfo::CreateTargetInfo(Clang->getDiagnostics(), &Clang->getTargetOpts())); + Clang->createVirtualFileSystem(); Clang->createFileManager(); Clang->createSourceManager(Clang->getFileManager()); Clang->createPreprocessor(); diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp index 61fe26c92227..0e9b0c0ebe30 100644 --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -194,9 +194,14 @@ CompilerInstance::createDiagnostics(DiagnosticOptions *Opts, return Diags; } +void CompilerInstance::createVirtualFileSystem() { + VirtualFileSystem = vfs::getRealFileSystem(); +} + // File Manager void CompilerInstance::createFileManager() { + assert(hasVirtualFileSystem() && "expected virtual file system"); FileMgr = new FileManager(getFileSystemOpts()); } @@ -867,6 +872,8 @@ static void compileModule(CompilerInstance &ImportingInstance, ImportingInstance.getDiagnosticClient()), /*ShouldOwnClient=*/true); + Instance.setVirtualFileSystem(&ImportingInstance.getVirtualFileSystem()); + // Note that this module is part of the module build stack, so that we // can detect cycles in the module graph. Instance.createFileManager(); // FIXME: Adopt file manager from importer? diff --git a/clang/lib/Frontend/FrontendAction.cpp b/clang/lib/Frontend/FrontendAction.cpp index 0baf3e5e1fb3..4e04c2c0e83a 100644 --- a/clang/lib/Frontend/FrontendAction.cpp +++ b/clang/lib/Frontend/FrontendAction.cpp @@ -159,7 +159,6 @@ ASTConsumer* FrontendAction::CreateWrappedASTConsumer(CompilerInstance &CI, return new MultiplexConsumer(Consumers); } - bool FrontendAction::BeginSourceFile(CompilerInstance &CI, const FrontendInputFile &Input) { assert(!Instance && "Already processing a source file!"); @@ -213,6 +212,8 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, } // Set up the file and source managers, if needed. + if (!CI.hasVirtualFileSystem()) + CI.createVirtualFileSystem(); if (!CI.hasFileManager()) CI.createFileManager(); if (!CI.hasSourceManager()) diff --git a/clang/lib/Lex/PTHLexer.cpp b/clang/lib/Lex/PTHLexer.cpp index e174222ece38..cdc5d7e33816 100644 --- a/clang/lib/Lex/PTHLexer.cpp +++ b/clang/lib/Lex/PTHLexer.cpp @@ -675,13 +675,13 @@ public: ~PTHStatCache() {} LookupResult getStat(const char *Path, FileData &Data, bool isFile, - int *FileDescriptor) { + vfs::File **F, vfs::FileSystem &FS) { // Do the lookup for the file's data in the PTH file. CacheTy::iterator I = Cache.find(Path); // If we don't get a hit in the PTH file just forward to 'stat'. if (I == Cache.end()) - return statChained(Path, Data, isFile, FileDescriptor); + return statChained(Path, Data, isFile, F, FS); const PTHStatData &D = *I; diff --git a/clang/lib/Serialization/ModuleManager.cpp b/clang/lib/Serialization/ModuleManager.cpp index 711afcd2dca1..ce7e7af3d138 100644 --- a/clang/lib/Serialization/ModuleManager.cpp +++ b/clang/lib/Serialization/ModuleManager.cpp @@ -89,7 +89,7 @@ ModuleManager::addModule(StringRef FileName, ModuleKind Type, New->InputFilesValidationTimestamp = 0; if (New->Kind == MK_Module) { std::string TimestampFilename = New->getTimestampFilename(); - llvm::sys::fs::file_status Status; + vfs::Status Status; // A cached stat value would be fine as well. if (!FileMgr.getNoncachedStatValue(TimestampFilename, Status)) New->InputFilesValidationTimestamp = diff --git a/clang/unittests/Basic/FileManagerTest.cpp b/clang/unittests/Basic/FileManagerTest.cpp index f8ce50d531e7..5b35e687350f 100644 --- a/clang/unittests/Basic/FileManagerTest.cpp +++ b/clang/unittests/Basic/FileManagerTest.cpp @@ -48,7 +48,7 @@ public: // Implement FileSystemStatCache::getStat(). virtual LookupResult getStat(const char *Path, FileData &Data, bool isFile, - int *FileDescriptor) { + vfs::File **F, vfs::FileSystem &FS) { if (StatCalls.count(Path) != 0) { Data = StatCalls[Path]; return CacheExists;