maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

Reapply [2] [VFS] Add 'overlay-relative' field to YAML files 

This reapplies r261552 and r263748. Fixed testcase to reapply.

The VFS overlay mapping between virtual paths and real paths is done through
the 'external-contents' entries in YAML files, which contains hardcoded paths
to the real files.

When a module compilation crashes, headers are dumped into <name>.cache/vfs
directory and are mapped via the <name>.cache/vfs/vfs.yaml. The script
generated for reproduction uses -ivfsoverlay pointing to file to gather the
mapping between virtual paths and files inside <name>.cache/vfs. Currently, we
are only capable of reproducing such crashes in the same machine as they
happen, because of the hardcoded paths in 'external-contents'.

To be able to reproduce a crash in another machine, this patch introduces a new
option in the VFS yaml file called 'overlay-relative'. When it's equal to
'true' it means that the provided path to the YAML file through the
-ivfsoverlay option should also be used to prefix the final path for every
'external-contents'.

Example, given the invocation snippet "... -ivfsoverlay
<name>.cache/vfs/vfs.yaml" and the following entry in the yaml file:

"overlay-relative": "true",
"roots": [
...
  "type": "directory",
  "name": "/usr/include",
  "contents": [
    {
      "type": "file",
      "name": "stdio.h",
      "external-contents": "/usr/include/stdio.h"
    },
...

Here, a file manager request for virtual "/usr/include/stdio.h", that will map
into real path "/<absolute_path_to>/<name>.cache/vfs/usr/include/stdio.h.

This is a useful feature for debugging module crashes in machines other than
the one where the error happened.

Differential Revision: http://reviews.llvm.org/D17457

rdar://problem/24499339

llvm-svn: 263893
This commit is contained in:
Bruno Cardoso Lopes 2016-03-20 02:08:48 +00:00
parent ea87eae4ca
commit d878e28e67
8 changed files with 175 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
clang/include/clang/Basic/VirtualFileSystem.h
View File

@ -310,6 +310,7 @@ llvm::sys::fs::UniqueID getNextVirtualUniqueID();
IntrusiveRefCntPtr<FileSystem>
getVFSFromYAML(std::unique_ptr<llvm::MemoryBuffer> Buffer,
llvm::SourceMgr::DiagHandlerTy DiagHandler,
StringRef YAMLFilePath,
void *DiagContext = nullptr,
IntrusiveRefCntPtr<FileSystem> ExternalFS = getRealFileSystem());
@ -323,6 +324,8 @@ struct YAMLVFSEntry {
class YAMLVFSWriter {
std::vector<YAMLVFSEntry> Mappings;
Optional<bool> IsCaseSensitive;
Optional<bool> IsOverlayRelative;
std::string OverlayDir;
public:
YAMLVFSWriter() {}
@ -330,6 +333,11 @@ public:
void setCaseSensitivity(bool CaseSensitive) {
IsCaseSensitive = CaseSensitive;
}
void setOverlayDir(StringRef OverlayDirectory) {
IsOverlayRelative = true;
OverlayDir.assign(OverlayDirectory.str());
}
void write(llvm::raw_ostream &OS);
};

113
clang/lib/Basic/VirtualFileSystem.cpp
View File

@ -780,6 +780,7 @@ public:
/// All configuration options are optional.
/// 'case-sensitive': <boolean, default=true>
/// 'use-external-names': <boolean, default=true>
/// 'overlay-relative': <boolean, default=false>
///
/// Virtual directories are represented as
/// \verbatim
@ -819,6 +820,10 @@ class RedirectingFileSystem : public vfs::FileSystem {
std::vector<std::unique_ptr<Entry>> Roots;
/// \brief The file system to use for external references.
IntrusiveRefCntPtr<FileSystem> ExternalFS;
/// If IsRelativeOverlay is set, this represents the directory
/// path that should be prefixed to each 'external-contents' entry
/// when reading from YAML files.
std::string ExternalContentsPrefixDir;
/// @name Configuration
/// @{
@ -828,6 +833,10 @@ class RedirectingFileSystem : public vfs::FileSystem {
/// Currently, case-insensitive matching only works correctly with ASCII.
bool CaseSensitive;
/// IsRelativeOverlay marks whether a IsExternalContentsPrefixDir path must
/// be prefixed in every 'external-contents' when reading from YAML files.
bool IsRelativeOverlay = false;
/// \brief Whether to use to use the value of 'external-contents' for the
/// names of files. This global value is overridable on a per-file basis.
bool UseExternalNames;
@ -865,8 +874,8 @@ public:
/// returns a virtual file system representing its contents.
static RedirectingFileSystem *
create(std::unique_ptr<MemoryBuffer> Buffer,
SourceMgr::DiagHandlerTy DiagHandler, void *DiagContext,
IntrusiveRefCntPtr<FileSystem> ExternalFS);
SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath,
void *DiagContext, IntrusiveRefCntPtr<FileSystem> ExternalFS);
ErrorOr<Status> status(const Twine &Path) override;
ErrorOr<std::unique_ptr<File>> openFileForRead(const Twine &Path) override;
@ -899,6 +908,15 @@ public:
return directory_iterator(std::make_shared<VFSFromYamlDirIterImpl>(Dir,
*this, D->contents_begin(), D->contents_end(), EC));
}
void setExternalContentsPrefixDir(StringRef PrefixDir) {
ExternalContentsPrefixDir = PrefixDir.str();
}
StringRef getExternalContentsPrefixDir() const {
return ExternalContentsPrefixDir;
}
};
/// \brief A helper class to hold the common YAML parsing state.
@ -1072,16 +1090,24 @@ class RedirectingFileSystemParser {
HasContents = true;
if (!parseScalarString(I->getValue(), Value, Buffer))
return nullptr;
SmallString<256> FullPath;
if (FS->IsRelativeOverlay) {
FullPath = FS->getExternalContentsPrefixDir();
assert(!FullPath.empty() &&
"External contents prefix directory must exist");
llvm::sys::path::append(FullPath, Value);
} else {
FullPath = Value;
}
if (FS->UseCanonicalizedPaths) {
SmallString<256> Path(Value);
// Guarantee that old YAML files containing paths with ".." and "."
// are properly canonicalized before read into the VFS.
Path = sys::path::remove_leading_dotslash(Path);
sys::path::remove_dots(Path, /*remove_dot_dot=*/true);
ExternalContentsPath = Path.str();
} else {
ExternalContentsPath = Value;
FullPath = sys::path::remove_leading_dotslash(FullPath);
sys::path::remove_dots(FullPath, /*remove_dot_dot=*/true);
}
ExternalContentsPath = FullPath.str();
} else if (Key == "use-external-name") {
bool Val;
if (!parseScalarBool(I->getValue(), Val))
@ -1167,6 +1193,7 @@ public:
KeyStatusPair("version", true),
KeyStatusPair("case-sensitive", false),
KeyStatusPair("use-external-names", false),
KeyStatusPair("overlay-relative", false),
KeyStatusPair("roots", true),
};
@ -1218,6 +1245,9 @@ public:
} else if (Key == "case-sensitive") {
if (!parseScalarBool(I->getValue(), FS->CaseSensitive))
return false;
} else if (Key == "overlay-relative") {
if (!parseScalarBool(I->getValue(), FS->IsRelativeOverlay))
return false;
} else if (Key == "use-external-names") {
if (!parseScalarBool(I->getValue(), FS->UseExternalNames))
return false;
@ -1238,9 +1268,11 @@ public:
Entry::~Entry() = default;
RedirectingFileSystem *RedirectingFileSystem::create(
std::unique_ptr<MemoryBuffer> Buffer, SourceMgr::DiagHandlerTy DiagHandler,
void *DiagContext, IntrusiveRefCntPtr<FileSystem> ExternalFS) {
RedirectingFileSystem *
RedirectingFileSystem::create(std::unique_ptr<MemoryBuffer> Buffer,
SourceMgr::DiagHandlerTy DiagHandler,
StringRef YAMLFilePath, void *DiagContext,
IntrusiveRefCntPtr<FileSystem> ExternalFS) {
SourceMgr SM;
yaml::Stream Stream(Buffer->getMemBufferRef(), SM);
@ -1257,6 +1289,23 @@ RedirectingFileSystem *RedirectingFileSystem::create(
std::unique_ptr<RedirectingFileSystem> FS(
new RedirectingFileSystem(ExternalFS));
if (!YAMLFilePath.empty()) {
// Use the YAML path from -ivfsoverlay to compute the dir to be prefixed
// to each 'external-contents' path.
//
// Example:
// -ivfsoverlay dummy.cache/vfs/vfs.yaml
// yields:
// FS->ExternalContentsPrefixDir => /<absolute_path_to>/dummy.cache/vfs
//
SmallString<256> OverlayAbsDir = sys::path::parent_path(YAMLFilePath);
std::error_code EC = llvm::sys::fs::make_absolute(OverlayAbsDir);
assert(!EC && "Overlay dir final path must be absolute");
(void)EC;
FS->setExternalContentsPrefixDir(OverlayAbsDir);
}
if (!P.parse(Root, FS.get()))
return nullptr;
@ -1410,10 +1459,12 @@ RedirectingFileSystem::openFileForRead(const Twine &Path) {
IntrusiveRefCntPtr<FileSystem>
vfs::getVFSFromYAML(std::unique_ptr<MemoryBuffer> Buffer,
SourceMgr::DiagHandlerTy DiagHandler, void *DiagContext,
SourceMgr::DiagHandlerTy DiagHandler,
StringRef YAMLFilePath,
void *DiagContext,
IntrusiveRefCntPtr<FileSystem> ExternalFS) {
return RedirectingFileSystem::create(std::move(Buffer), DiagHandler,
DiagContext, ExternalFS);
YAMLFilePath, DiagContext, ExternalFS);
}
UniqueID vfs::getNextVirtualUniqueID() {
@ -1445,7 +1496,8 @@ class JSONWriter {
public:
JSONWriter(llvm::raw_ostream &OS) : OS(OS) {}
void write(ArrayRef<YAMLVFSEntry> Entries, Optional<bool> IsCaseSensitive);
void write(ArrayRef<YAMLVFSEntry> Entries, Optional<bool> IsCaseSensitive,
Optional<bool> IsOverlayRelative, StringRef OverlayDir);
};
}
@ -1498,7 +1550,9 @@ void JSONWriter::writeEntry(StringRef VPath, StringRef RPath) {
}
void JSONWriter::write(ArrayRef<YAMLVFSEntry> Entries,
Optional<bool> IsCaseSensitive) {
Optional<bool> IsCaseSensitive,
Optional<bool> IsOverlayRelative,
StringRef OverlayDir) {
using namespace llvm::sys;
OS << "{\n"
@ -1506,12 +1560,27 @@ void JSONWriter::write(ArrayRef<YAMLVFSEntry> Entries,
if (IsCaseSensitive.hasValue())
OS << " 'case-sensitive': '"
<< (IsCaseSensitive.getValue() ? "true" : "false") << "',\n";
bool UseOverlayRelative = false;
if (IsOverlayRelative.hasValue()) {
UseOverlayRelative = IsOverlayRelative.getValue();
OS << " 'overlay-relative': '"
<< (UseOverlayRelative ? "true" : "false") << "',\n";
}
OS << " 'roots': [\n";
if (!Entries.empty()) {
const YAMLVFSEntry &Entry = Entries.front();
startDirectory(path::parent_path(Entry.VPath));
writeEntry(path::filename(Entry.VPath), Entry.RPath);
StringRef RPath = Entry.RPath;
if (UseOverlayRelative) {
unsigned OverlayDirLen = OverlayDir.size();
assert(RPath.substr(0, OverlayDirLen) == OverlayDir &&
"Overlay dir must be contained in RPath");
RPath = RPath.slice(OverlayDirLen, RPath.size());
}
writeEntry(path::filename(Entry.VPath), RPath);
for (const auto &Entry : Entries.slice(1)) {
StringRef Dir = path::parent_path(Entry.VPath);
@ -1525,7 +1594,14 @@ void JSONWriter::write(ArrayRef<YAMLVFSEntry> Entries,
OS << ",\n";
startDirectory(Dir);
}
writeEntry(path::filename(Entry.VPath), Entry.RPath);
StringRef RPath = Entry.RPath;
if (UseOverlayRelative) {
unsigned OverlayDirLen = OverlayDir.size();
assert(RPath.substr(0, OverlayDirLen) == OverlayDir &&
"Overlay dir must be contained in RPath");
RPath = RPath.slice(OverlayDirLen, RPath.size());
}
writeEntry(path::filename(Entry.VPath), RPath);
}
while (!DirStack.empty()) {
@ -1545,7 +1621,8 @@ void YAMLVFSWriter::write(llvm::raw_ostream &OS) {
return LHS.VPath < RHS.VPath;
});
JSONWriter(OS).write(Mappings, IsCaseSensitive);
JSONWriter(OS).write(Mappings, IsCaseSensitive, IsOverlayRelative,
OverlayDir);
}
VFSFromYamlDirIterImpl::VFSFromYamlDirIterImpl(

4
clang/lib/Frontend/CompilerInvocation.cpp
View File

@ -2383,8 +2383,8 @@ createVFSFromCompilerInvocation(const CompilerInvocation &CI,
return IntrusiveRefCntPtr<vfs::FileSystem>();
}
IntrusiveRefCntPtr<vfs::FileSystem> FS =
vfs::getVFSFromYAML(std::move(Buffer.get()), /*DiagHandler*/ nullptr);
IntrusiveRefCntPtr<vfs::FileSystem> FS = vfs::getVFSFromYAML(
std::move(Buffer.get()), /*DiagHandler*/ nullptr, File);
if (!FS.get()) {
Diags.Report(diag::err_invalid_vfs_overlay) << File;
return IntrusiveRefCntPtr<vfs::FileSystem>();

4
clang/lib/Frontend/ModuleDependencyCollector.cpp
View File

@ -50,6 +50,10 @@ void ModuleDependencyCollector::writeFileMap() {
SmallString<256> Dest = getDest();
llvm::sys::path::append(Dest, "vfs.yaml");
// Default to use relative overlay directories in the VFS yaml file. This
// allows crash reproducer scripts to work across machines.
VFSWriter.setOverlayDir(getDest());
std::error_code EC;
llvm::raw_fd_ostream OS(Dest, EC, llvm::sys::fs::F_Text);
if (EC) {

8
clang/test/Modules/crash-vfs-path-symlink-component.m
View File

@ -40,21 +40,21 @@
// CHECKSH: "-ivfsoverlay" "crash-vfs-{{[^ ]*}}.cache/vfs/vfs.yaml"
// CHECKYAML: 'type': 'directory'
// CHECKYAML: 'name': "{{[^ ]*}}/i/usr/include",
// CHECKYAML: 'name': "/[[PATH:.*]]/i/usr/include",
// CHECKYAML-NEXT: 'contents': [
// CHECKYAML-NEXT: {
// CHECKYAML-NEXT: 'type': 'file',
// CHECKYAML-NEXT: 'name': "module.map",
// CHECKYAML-NEXT: 'external-contents': "{{[^ ]*}}.cache/vfs/{{[^ ]*}}/i/usr/include/module.map"
// CHECKYAML-NEXT: 'external-contents': "/[[PATH]]/i/usr/include/module.map"
// CHECKYAML-NEXT: },
// CHECKYAML: 'type': 'directory'
// CHECKYAML: 'name': "{{[^ ]*}}/i/usr",
// CHECKYAML: 'name': "/[[PATH]]/i/usr",
// CHECKYAML-NEXT: 'contents': [
// CHECKYAML-NEXT: {
// CHECKYAML-NEXT: 'type': 'file',
// CHECKYAML-NEXT: 'name': "module.map",
// CHECKYAML-NEXT: 'external-contents': "{{[^ ]*}}.cache/vfs/{{[^ ]*}}/i/usr/include/module.map"
// CHECKYAML-NEXT: 'external-contents': "/[[PATH]]/i/usr/include/module.map"
// CHECKYAML-NEXT: },
// Test that by using the previous generated YAML file clang is able to find the

9
clang/test/Modules/crash-vfs-path-traversal.m
View File

@ -36,13 +36,13 @@
// CHECKSH: "crash-vfs-{{[^ ]*}}.m"
// CHECKSH: "-ivfsoverlay" "crash-vfs-{{[^ ]*}}.cache/vfs/vfs.yaml"
// CHECKYAML: 'type': 'directory'
// CHECKYAML: 'name': "{{[^ ]*}}/Inputs/crash-recovery/usr/include",
// CHECKYAML: 'type': 'directory'
// CHECKYAML: 'name': "/[[PATH:.*]]/Inputs/crash-recovery/usr/include",
// CHECKYAML-NEXT: 'contents': [
// CHECKYAML-NEXT: {
// CHECKYAML-NEXT: 'type': 'file',
// CHECKYAML-NEXT: 'name': "module.map",
// CHECKYAML-NEXT: 'external-contents': "{{[^ ]*}}/Inputs/crash-recovery/usr/include/module.map"
// CHECKYAML-NEXT: 'external-contents': "/[[PATH]]/Inputs/crash-recovery/usr/include/module.map"
// CHECKYAML-NEXT: },
// Replace the paths in the YAML files with relative ".." traversals
@ -51,9 +51,10 @@
// RUN: sed -e "s@usr/include@usr/include/../include@g" \
// RUN: %t/crash-vfs-*.cache/vfs/vfs.yaml > %t/vfs.yaml
// RUN: cp %t/vfs.yaml %t/crash-vfs-*.cache/vfs/vfs.yaml
// RUN: unset FORCE_CLANG_DIAGNOSTICS_CRASH
// RUN: %clang -E %s -I %S/Inputs/crash-recovery -isysroot %/t/i/ \
// RUN: -ivfsoverlay %t/vfs.yaml -fmodules \
// RUN: -ivfsoverlay %t/crash-vfs-*.cache/vfs/vfs.yaml -fmodules \
// RUN: -fmodules-cache-path=%t/m/ 2>&1 \
// RUN: | FileCheck %s --check-prefix=CHECKOVERLAY

56
clang/test/Modules/crash-vfs-relative-overlay.m Normal file
View File

@ -0,0 +1,56 @@
// REQUIRES: crash-recovery, shell
// FIXME: This XFAIL is cargo-culted from crash-report.c. Do we need it?
// XFAIL: mingw32
// RUN: rm -rf %t
// RUN: mkdir -p %t/i %t/m %t
// RUN: not env FORCE_CLANG_DIAGNOSTICS_CRASH= TMPDIR=%t TEMP=%t TMP=%t \
// RUN: %clang -fsyntax-only %s -I %S/Inputs/System -isysroot %/t/i/ \
// RUN: -fmodules -fmodules-cache-path=%t/m/ 2>&1 | FileCheck %s
// RUN: FileCheck --check-prefix=CHECKSRC %s -input-file %t/crash-vfs-*.m
// RUN: FileCheck --check-prefix=CHECKSH %s -input-file %t/crash-vfs-*.sh
// RUN: FileCheck --check-prefix=CHECKYAML %s -input-file \
// RUN: %t/crash-vfs-*.cache/vfs/vfs.yaml
// RUN: find %t/crash-vfs-*.cache/vfs | \
// RUN: grep "Inputs/System/usr/include/stdio.h" | count 1
#include "usr/include/stdio.h"
// CHECK: Preprocessed source(s) and associated run script(s) are located at:
// CHECK-NEXT: note: diagnostic msg: {{.*}}.m
// CHECK-NEXT: note: diagnostic msg: {{.*}}.cache
// CHECKSRC: @import cstd.stdio;
// CHECKSH: # Crash reproducer
// CHECKSH-NEXT: # Driver args: "-fsyntax-only"
// CHECKSH-NEXT: # Original command: {{.*$}}
// CHECKSH-NEXT: "-cc1"
// CHECKSH: "-isysroot" "{{[^"]*}}/i/"
// CHECKSH-NOT: "-fmodules-cache-path="
// CHECKSH: "crash-vfs-{{[^ ]*}}.m"
// CHECKSH: "-ivfsoverlay" "crash-vfs-{{[^ ]*}}.cache/vfs/vfs.yaml"
// CHECKYAML: 'type': 'directory'
// CHECKYAML: 'name': "/[[PATH:.*]]/Inputs/System/usr/include",
// CHECKYAML-NEXT: 'contents': [
// CHECKYAML-NEXT: {
// CHECKYAML-NEXT: 'type': 'file',
// CHECKYAML-NEXT: 'name': "module.map",
// CHECKYAML-NOT: 'external-contents': "{{[^ ]*}}.cache
// CHECKYAML-NEXT: 'external-contents': "/[[PATH]]/Inputs/System/usr/include/module.map"
// CHECKYAML-NEXT: },
// Test that reading the YAML file will yield the correct path after
// the overlay dir is prefixed to access headers in .cache/vfs directory.
// RUN: unset FORCE_CLANG_DIAGNOSTICS_CRASH
// RUN: %clang -E %s -I %S/Inputs/System -isysroot %/t/i/ \
// RUN: -ivfsoverlay %t/crash-vfs-*.cache/vfs/vfs.yaml -fmodules \
// RUN: -fmodules-cache-path=%t/m/ 2>&1 \
// RUN: | FileCheck %s --check-prefix=CHECKOVERLAY
// CHECKOVERLAY: @import cstd.stdio; /* clang -E: implicit import for "{{[^ ]*}}.cache/vfs/{{[^ ]*}}/usr/include/stdio.h"

2
clang/unittests/Basic/VirtualFileSystemTest.cpp
View File

@ -662,7 +662,7 @@ public:
getFromYAMLRawString(StringRef Content,
IntrusiveRefCntPtr<vfs::FileSystem> ExternalFS) {
std::unique_ptr<MemoryBuffer> Buffer = MemoryBuffer::getMemBuffer(Content);
return getVFSFromYAML(std::move(Buffer), CountingDiagHandler, this,
return getVFSFromYAML(std::move(Buffer), CountingDiagHandler, "", this,
ExternalFS);
}