2012-11-02 01:52:58 +08:00
|
|
|
//===- unittests/Lex/PPCallbacksTest.cpp - PPCallbacks tests ------===//
|
|
|
|
//
|
|
|
|
// The LLVM Compiler Infrastructure
|
|
|
|
//
|
|
|
|
// This file is distributed under the University of Illinois Open Source
|
|
|
|
// License. See LICENSE.TXT for details.
|
|
|
|
//
|
|
|
|
//===--------------------------------------------------------------===//
|
|
|
|
|
2012-12-04 17:45:34 +08:00
|
|
|
#include "clang/Lex/Preprocessor.h"
|
2014-01-07 19:51:46 +08:00
|
|
|
#include "clang/AST/ASTConsumer.h"
|
|
|
|
#include "clang/AST/ASTContext.h"
|
2012-11-02 01:52:58 +08:00
|
|
|
#include "clang/Basic/Diagnostic.h"
|
2014-05-11 00:31:55 +08:00
|
|
|
#include "clang/Basic/DiagnosticOptions.h"
|
2012-11-02 01:52:58 +08:00
|
|
|
#include "clang/Basic/FileManager.h"
|
|
|
|
#include "clang/Basic/LangOptions.h"
|
Reapply "Modules: Cache PCMs in memory and avoid a use-after-free"
This reverts commit r298185, effectively reapplying r298165, after fixing the
new unit tests (PR32338). The memory buffer generator doesn't null-terminate
the MemoryBuffer it creates; this version of the commit informs getMemBuffer
about that to avoid the assert.
Original commit message follows:
----
Clang's internal build system for implicit modules uses lock files to
ensure that after a process writes a PCM it will read the same one back
in (without contention from other -cc1 commands). Since PCMs are read
from disk repeatedly while invalidating, building, and importing, the
lock is not released quickly. Furthermore, the LockFileManager is not
robust in every environment. Other -cc1 commands can stall until
timeout (after about eight minutes).
This commit changes the lock file from being necessary for correctness
to a (possibly dubious) performance hack. The remaining benefit is to
reduce duplicate work in competing -cc1 commands which depend on the
same module. Follow-up commits will change the internal build system to
continue after a timeout, and reduce the timeout. Perhaps we should
reconsider blocking at all.
This also fixes a use-after-free, when one part of a compilation
validates a PCM and starts using it, and another tries to swap out the
PCM for something new.
The PCMCache is a new type called MemoryBufferCache, which saves memory
buffers based on their filename. Its ownership is shared by the
CompilerInstance and ModuleManager.
- The ModuleManager stores PCMs there that it loads from disk, never
touching the disk if the cache is hot.
- When modules fail to validate, they're removed from the cache.
- When a CompilerInstance is spawned to build a new module, each
already-loaded PCM is assumed to be valid, and is frozen to avoid
the use-after-free.
- Any newly-built module is written directly to the cache to avoid the
round-trip to the filesystem, making lock files unnecessary for
correctness.
Original patch by Manman Ren; most testcases by Adrian Prantl!
llvm-svn: 298278
2017-03-21 01:58:26 +08:00
|
|
|
#include "clang/Basic/MemoryBufferCache.h"
|
2012-11-02 01:52:58 +08:00
|
|
|
#include "clang/Basic/SourceManager.h"
|
|
|
|
#include "clang/Basic/TargetInfo.h"
|
|
|
|
#include "clang/Basic/TargetOptions.h"
|
|
|
|
#include "clang/Lex/HeaderSearch.h"
|
|
|
|
#include "clang/Lex/HeaderSearchOptions.h"
|
|
|
|
#include "clang/Lex/ModuleLoader.h"
|
|
|
|
#include "clang/Lex/PreprocessorOptions.h"
|
2013-10-12 17:29:48 +08:00
|
|
|
#include "clang/Parse/Parser.h"
|
|
|
|
#include "clang/Sema/Sema.h"
|
2012-11-02 01:52:58 +08:00
|
|
|
#include "llvm/ADT/SmallString.h"
|
2013-06-12 06:15:02 +08:00
|
|
|
#include "llvm/Support/Path.h"
|
2012-11-02 01:52:58 +08:00
|
|
|
#include "gtest/gtest.h"
|
|
|
|
|
|
|
|
using namespace clang;
|
|
|
|
|
|
|
|
namespace {
|
|
|
|
|
|
|
|
// Stub to collect data from InclusionDirective callbacks.
|
|
|
|
class InclusionDirectiveCallbacks : public PPCallbacks {
|
|
|
|
public:
|
2015-04-11 10:00:23 +08:00
|
|
|
void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
|
|
|
|
StringRef FileName, bool IsAngled,
|
|
|
|
CharSourceRange FilenameRange, const FileEntry *File,
|
|
|
|
StringRef SearchPath, StringRef RelativePath,
|
2018-05-11 03:05:36 +08:00
|
|
|
const Module *Imported,
|
|
|
|
SrcMgr::CharacteristicKind FileType) override {
|
|
|
|
this->HashLoc = HashLoc;
|
|
|
|
this->IncludeTok = IncludeTok;
|
|
|
|
this->FileName = FileName.str();
|
|
|
|
this->IsAngled = IsAngled;
|
|
|
|
this->FilenameRange = FilenameRange;
|
|
|
|
this->File = File;
|
|
|
|
this->SearchPath = SearchPath.str();
|
|
|
|
this->RelativePath = RelativePath.str();
|
|
|
|
this->Imported = Imported;
|
|
|
|
this->FileType = FileType;
|
2012-11-02 01:52:58 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
SourceLocation HashLoc;
|
|
|
|
Token IncludeTok;
|
|
|
|
SmallString<16> FileName;
|
|
|
|
bool IsAngled;
|
|
|
|
CharSourceRange FilenameRange;
|
|
|
|
const FileEntry* File;
|
|
|
|
SmallString<16> SearchPath;
|
|
|
|
SmallString<16> RelativePath;
|
|
|
|
const Module* Imported;
|
2018-05-11 03:05:36 +08:00
|
|
|
SrcMgr::CharacteristicKind FileType;
|
2012-11-02 01:52:58 +08:00
|
|
|
};
|
|
|
|
|
2019-01-11 05:22:13 +08:00
|
|
|
class CondDirectiveCallbacks : public PPCallbacks {
|
|
|
|
public:
|
|
|
|
struct Result {
|
|
|
|
SourceRange ConditionRange;
|
|
|
|
ConditionValueKind ConditionValue;
|
|
|
|
|
|
|
|
Result(SourceRange R, ConditionValueKind K)
|
|
|
|
: ConditionRange(R), ConditionValue(K) {}
|
|
|
|
};
|
|
|
|
|
|
|
|
std::vector<Result> Results;
|
|
|
|
|
|
|
|
void If(SourceLocation Loc, SourceRange ConditionRange,
|
|
|
|
ConditionValueKind ConditionValue) override {
|
|
|
|
Results.emplace_back(ConditionRange, ConditionValue);
|
|
|
|
}
|
|
|
|
|
|
|
|
void Elif(SourceLocation Loc, SourceRange ConditionRange,
|
|
|
|
ConditionValueKind ConditionValue, SourceLocation IfLoc) override {
|
|
|
|
Results.emplace_back(ConditionRange, ConditionValue);
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2013-10-12 17:29:48 +08:00
|
|
|
// Stub to collect data from PragmaOpenCLExtension callbacks.
|
|
|
|
class PragmaOpenCLExtensionCallbacks : public PPCallbacks {
|
|
|
|
public:
|
|
|
|
typedef struct {
|
2013-10-14 15:13:59 +08:00
|
|
|
SmallString<16> Name;
|
2013-10-12 17:29:48 +08:00
|
|
|
unsigned State;
|
|
|
|
} CallbackParameters;
|
|
|
|
|
2015-07-23 04:46:26 +08:00
|
|
|
PragmaOpenCLExtensionCallbacks() : Name("Not called."), State(99) {}
|
2013-10-12 17:29:48 +08:00
|
|
|
|
2015-04-11 10:00:23 +08:00
|
|
|
void PragmaOpenCLExtension(clang::SourceLocation NameLoc,
|
|
|
|
const clang::IdentifierInfo *Name,
|
|
|
|
clang::SourceLocation StateLoc,
|
|
|
|
unsigned State) override {
|
2013-10-12 17:29:48 +08:00
|
|
|
this->NameLoc = NameLoc;
|
2013-10-14 15:13:59 +08:00
|
|
|
this->Name = Name->getName();
|
2013-10-12 17:29:48 +08:00
|
|
|
this->StateLoc = StateLoc;
|
|
|
|
this->State = State;
|
2015-07-23 04:46:26 +08:00
|
|
|
}
|
2013-10-12 17:29:48 +08:00
|
|
|
|
|
|
|
SourceLocation NameLoc;
|
2013-10-14 15:13:59 +08:00
|
|
|
SmallString<16> Name;
|
2013-10-12 17:29:48 +08:00
|
|
|
SourceLocation StateLoc;
|
|
|
|
unsigned State;
|
|
|
|
};
|
|
|
|
|
2012-11-02 01:52:58 +08:00
|
|
|
// PPCallbacks test fixture.
|
|
|
|
class PPCallbacksTest : public ::testing::Test {
|
|
|
|
protected:
|
|
|
|
PPCallbacksTest()
|
2018-10-10 21:27:25 +08:00
|
|
|
: InMemoryFileSystem(new llvm::vfs::InMemoryFileSystem),
|
2015-10-08 22:20:14 +08:00
|
|
|
FileMgr(FileSystemOptions(), InMemoryFileSystem),
|
|
|
|
DiagID(new DiagnosticIDs()), DiagOpts(new DiagnosticOptions()),
|
2014-07-06 13:26:44 +08:00
|
|
|
Diags(DiagID, DiagOpts.get(), new IgnoringDiagConsumer()),
|
|
|
|
SourceMgr(Diags, FileMgr), TargetOpts(new TargetOptions()) {
|
2012-11-02 01:52:58 +08:00
|
|
|
TargetOpts->Triple = "x86_64-apple-darwin11.1.0";
|
2014-07-06 13:26:44 +08:00
|
|
|
Target = TargetInfo::CreateTargetInfo(Diags, TargetOpts);
|
2012-11-02 01:52:58 +08:00
|
|
|
}
|
|
|
|
|
2018-10-10 21:27:25 +08:00
|
|
|
IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> InMemoryFileSystem;
|
2012-11-02 01:52:58 +08:00
|
|
|
FileManager FileMgr;
|
|
|
|
IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
|
|
|
|
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts;
|
|
|
|
DiagnosticsEngine Diags;
|
|
|
|
SourceManager SourceMgr;
|
|
|
|
LangOptions LangOpts;
|
2014-07-06 13:26:44 +08:00
|
|
|
std::shared_ptr<TargetOptions> TargetOpts;
|
2012-11-02 01:52:58 +08:00
|
|
|
IntrusiveRefCntPtr<TargetInfo> Target;
|
|
|
|
|
|
|
|
// Register a header path as a known file and add its location
|
|
|
|
// to search path.
|
2018-10-10 21:27:25 +08:00
|
|
|
void AddFakeHeader(HeaderSearch &HeaderInfo, const char *HeaderPath,
|
|
|
|
bool IsSystemHeader) {
|
|
|
|
// Tell FileMgr about header.
|
|
|
|
InMemoryFileSystem->addFile(HeaderPath, 0,
|
|
|
|
llvm::MemoryBuffer::getMemBuffer("\n"));
|
|
|
|
|
|
|
|
// Add header's parent path to search path.
|
|
|
|
StringRef SearchPath = llvm::sys::path::parent_path(HeaderPath);
|
|
|
|
const DirectoryEntry *DE = FileMgr.getDirectory(SearchPath);
|
|
|
|
DirectoryLookup DL(DE, SrcMgr::C_User, false);
|
|
|
|
HeaderInfo.AddSearchPath(DL, IsSystemHeader);
|
2012-11-02 01:52:58 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// Get the raw source string of the range.
|
|
|
|
StringRef GetSourceString(CharSourceRange Range) {
|
|
|
|
const char* B = SourceMgr.getCharacterData(Range.getBegin());
|
|
|
|
const char* E = SourceMgr.getCharacterData(Range.getEnd());
|
|
|
|
|
|
|
|
return StringRef(B, E - B);
|
|
|
|
}
|
|
|
|
|
2019-01-11 05:22:13 +08:00
|
|
|
StringRef GetSourceStringToEnd(CharSourceRange Range) {
|
|
|
|
const char *B = SourceMgr.getCharacterData(Range.getBegin());
|
|
|
|
const char *E = SourceMgr.getCharacterData(Range.getEnd());
|
|
|
|
|
|
|
|
return StringRef(
|
|
|
|
B,
|
|
|
|
E - B + Lexer::MeasureTokenLength(Range.getEnd(), SourceMgr, LangOpts));
|
|
|
|
}
|
|
|
|
|
2012-11-02 01:52:58 +08:00
|
|
|
// Run lexer over SourceText and collect FilenameRange from
|
|
|
|
// the InclusionDirective callback.
|
2018-10-10 21:27:25 +08:00
|
|
|
CharSourceRange InclusionDirectiveFilenameRange(const char *SourceText,
|
|
|
|
const char *HeaderPath,
|
|
|
|
bool SystemHeader) {
|
2014-10-16 06:00:40 +08:00
|
|
|
std::unique_ptr<llvm::MemoryBuffer> Buf =
|
|
|
|
llvm::MemoryBuffer::getMemBuffer(SourceText);
|
2014-08-29 15:59:55 +08:00
|
|
|
SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf)));
|
2012-11-02 01:52:58 +08:00
|
|
|
|
2017-06-10 03:22:32 +08:00
|
|
|
TrivialModuleLoader ModLoader;
|
Reapply "Modules: Cache PCMs in memory and avoid a use-after-free"
This reverts commit r298185, effectively reapplying r298165, after fixing the
new unit tests (PR32338). The memory buffer generator doesn't null-terminate
the MemoryBuffer it creates; this version of the commit informs getMemBuffer
about that to avoid the assert.
Original commit message follows:
----
Clang's internal build system for implicit modules uses lock files to
ensure that after a process writes a PCM it will read the same one back
in (without contention from other -cc1 commands). Since PCMs are read
from disk repeatedly while invalidating, building, and importing, the
lock is not released quickly. Furthermore, the LockFileManager is not
robust in every environment. Other -cc1 commands can stall until
timeout (after about eight minutes).
This commit changes the lock file from being necessary for correctness
to a (possibly dubious) performance hack. The remaining benefit is to
reduce duplicate work in competing -cc1 commands which depend on the
same module. Follow-up commits will change the internal build system to
continue after a timeout, and reduce the timeout. Perhaps we should
reconsider blocking at all.
This also fixes a use-after-free, when one part of a compilation
validates a PCM and starts using it, and another tries to swap out the
PCM for something new.
The PCMCache is a new type called MemoryBufferCache, which saves memory
buffers based on their filename. Its ownership is shared by the
CompilerInstance and ModuleManager.
- The ModuleManager stores PCMs there that it loads from disk, never
touching the disk if the cache is hot.
- When modules fail to validate, they're removed from the cache.
- When a CompilerInstance is spawned to build a new module, each
already-loaded PCM is assumed to be valid, and is frozen to avoid
the use-after-free.
- Any newly-built module is written directly to the cache to avoid the
round-trip to the filesystem, making lock files unnecessary for
correctness.
Original patch by Manman Ren; most testcases by Adrian Prantl!
llvm-svn: 298278
2017-03-21 01:58:26 +08:00
|
|
|
MemoryBufferCache PCMCache;
|
2012-11-02 01:52:58 +08:00
|
|
|
|
2017-01-06 09:04:46 +08:00
|
|
|
HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr,
|
|
|
|
Diags, LangOpts, Target.get());
|
2012-11-02 01:52:58 +08:00
|
|
|
AddFakeHeader(HeaderInfo, HeaderPath, SystemHeader);
|
|
|
|
|
2017-01-06 03:11:36 +08:00
|
|
|
Preprocessor PP(std::make_shared<PreprocessorOptions>(), Diags, LangOpts,
|
Reapply "Modules: Cache PCMs in memory and avoid a use-after-free"
This reverts commit r298185, effectively reapplying r298165, after fixing the
new unit tests (PR32338). The memory buffer generator doesn't null-terminate
the MemoryBuffer it creates; this version of the commit informs getMemBuffer
about that to avoid the assert.
Original commit message follows:
----
Clang's internal build system for implicit modules uses lock files to
ensure that after a process writes a PCM it will read the same one back
in (without contention from other -cc1 commands). Since PCMs are read
from disk repeatedly while invalidating, building, and importing, the
lock is not released quickly. Furthermore, the LockFileManager is not
robust in every environment. Other -cc1 commands can stall until
timeout (after about eight minutes).
This commit changes the lock file from being necessary for correctness
to a (possibly dubious) performance hack. The remaining benefit is to
reduce duplicate work in competing -cc1 commands which depend on the
same module. Follow-up commits will change the internal build system to
continue after a timeout, and reduce the timeout. Perhaps we should
reconsider blocking at all.
This also fixes a use-after-free, when one part of a compilation
validates a PCM and starts using it, and another tries to swap out the
PCM for something new.
The PCMCache is a new type called MemoryBufferCache, which saves memory
buffers based on their filename. Its ownership is shared by the
CompilerInstance and ModuleManager.
- The ModuleManager stores PCMs there that it loads from disk, never
touching the disk if the cache is hot.
- When modules fail to validate, they're removed from the cache.
- When a CompilerInstance is spawned to build a new module, each
already-loaded PCM is assumed to be valid, and is frozen to avoid
the use-after-free.
- Any newly-built module is written directly to the cache to avoid the
round-trip to the filesystem, making lock files unnecessary for
correctness.
Original patch by Manman Ren; most testcases by Adrian Prantl!
llvm-svn: 298278
2017-03-21 01:58:26 +08:00
|
|
|
SourceMgr, PCMCache, HeaderInfo, ModLoader,
|
2014-06-08 16:38:12 +08:00
|
|
|
/*IILookup =*/nullptr,
|
2014-05-02 11:43:38 +08:00
|
|
|
/*OwnsHeaderSearch =*/false);
|
2018-05-11 03:05:36 +08:00
|
|
|
return InclusionDirectiveCallback(PP)->FilenameRange;
|
|
|
|
}
|
|
|
|
|
|
|
|
SrcMgr::CharacteristicKind InclusionDirectiveCharacteristicKind(
|
|
|
|
const char *SourceText, const char *HeaderPath, bool SystemHeader) {
|
|
|
|
std::unique_ptr<llvm::MemoryBuffer> Buf =
|
|
|
|
llvm::MemoryBuffer::getMemBuffer(SourceText);
|
|
|
|
SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf)));
|
|
|
|
|
|
|
|
TrivialModuleLoader ModLoader;
|
|
|
|
MemoryBufferCache PCMCache;
|
|
|
|
|
|
|
|
HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr,
|
|
|
|
Diags, LangOpts, Target.get());
|
|
|
|
AddFakeHeader(HeaderInfo, HeaderPath, SystemHeader);
|
|
|
|
|
|
|
|
Preprocessor PP(std::make_shared<PreprocessorOptions>(), Diags, LangOpts,
|
|
|
|
SourceMgr, PCMCache, HeaderInfo, ModLoader,
|
|
|
|
/*IILookup =*/nullptr,
|
|
|
|
/*OwnsHeaderSearch =*/false);
|
|
|
|
return InclusionDirectiveCallback(PP)->FileType;
|
|
|
|
}
|
|
|
|
|
|
|
|
InclusionDirectiveCallbacks *InclusionDirectiveCallback(Preprocessor &PP) {
|
2014-05-02 11:43:30 +08:00
|
|
|
PP.Initialize(*Target);
|
2012-11-02 01:52:58 +08:00
|
|
|
InclusionDirectiveCallbacks* Callbacks = new InclusionDirectiveCallbacks;
|
2014-09-10 12:53:53 +08:00
|
|
|
PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(Callbacks));
|
2012-11-02 01:52:58 +08:00
|
|
|
|
|
|
|
// Lex source text.
|
|
|
|
PP.EnterMainSourceFile();
|
|
|
|
|
|
|
|
while (true) {
|
|
|
|
Token Tok;
|
|
|
|
PP.Lex(Tok);
|
|
|
|
if (Tok.is(tok::eof))
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Callbacks have been executed at this point -- return filename range.
|
2018-05-11 03:05:36 +08:00
|
|
|
return Callbacks;
|
2012-11-02 01:52:58 +08:00
|
|
|
}
|
2013-10-12 17:29:48 +08:00
|
|
|
|
2019-01-11 05:22:13 +08:00
|
|
|
std::vector<CondDirectiveCallbacks::Result>
|
|
|
|
DirectiveExprRange(StringRef SourceText) {
|
|
|
|
TrivialModuleLoader ModLoader;
|
|
|
|
MemoryBufferCache PCMCache;
|
|
|
|
std::unique_ptr<llvm::MemoryBuffer> Buf =
|
|
|
|
llvm::MemoryBuffer::getMemBuffer(SourceText);
|
|
|
|
SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf)));
|
|
|
|
HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr,
|
|
|
|
Diags, LangOpts, Target.get());
|
|
|
|
Preprocessor PP(std::make_shared<PreprocessorOptions>(), Diags, LangOpts,
|
|
|
|
SourceMgr, PCMCache, HeaderInfo, ModLoader,
|
|
|
|
/*IILookup =*/nullptr,
|
|
|
|
/*OwnsHeaderSearch =*/false);
|
|
|
|
PP.Initialize(*Target);
|
|
|
|
auto *Callbacks = new CondDirectiveCallbacks;
|
|
|
|
PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(Callbacks));
|
|
|
|
|
|
|
|
// Lex source text.
|
|
|
|
PP.EnterMainSourceFile();
|
|
|
|
|
|
|
|
while (true) {
|
|
|
|
Token Tok;
|
|
|
|
PP.Lex(Tok);
|
|
|
|
if (Tok.is(tok::eof))
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return Callbacks->Results;
|
|
|
|
}
|
|
|
|
|
2018-10-10 21:27:25 +08:00
|
|
|
PragmaOpenCLExtensionCallbacks::CallbackParameters
|
|
|
|
PragmaOpenCLExtensionCall(const char *SourceText) {
|
2013-10-12 17:29:48 +08:00
|
|
|
LangOptions OpenCLLangOpts;
|
|
|
|
OpenCLLangOpts.OpenCL = 1;
|
|
|
|
|
2014-10-16 06:00:40 +08:00
|
|
|
std::unique_ptr<llvm::MemoryBuffer> SourceBuf =
|
|
|
|
llvm::MemoryBuffer::getMemBuffer(SourceText, "test.cl");
|
2014-08-29 15:59:55 +08:00
|
|
|
SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(SourceBuf)));
|
2013-10-12 17:29:48 +08:00
|
|
|
|
2017-06-10 03:22:32 +08:00
|
|
|
TrivialModuleLoader ModLoader;
|
Reapply "Modules: Cache PCMs in memory and avoid a use-after-free"
This reverts commit r298185, effectively reapplying r298165, after fixing the
new unit tests (PR32338). The memory buffer generator doesn't null-terminate
the MemoryBuffer it creates; this version of the commit informs getMemBuffer
about that to avoid the assert.
Original commit message follows:
----
Clang's internal build system for implicit modules uses lock files to
ensure that after a process writes a PCM it will read the same one back
in (without contention from other -cc1 commands). Since PCMs are read
from disk repeatedly while invalidating, building, and importing, the
lock is not released quickly. Furthermore, the LockFileManager is not
robust in every environment. Other -cc1 commands can stall until
timeout (after about eight minutes).
This commit changes the lock file from being necessary for correctness
to a (possibly dubious) performance hack. The remaining benefit is to
reduce duplicate work in competing -cc1 commands which depend on the
same module. Follow-up commits will change the internal build system to
continue after a timeout, and reduce the timeout. Perhaps we should
reconsider blocking at all.
This also fixes a use-after-free, when one part of a compilation
validates a PCM and starts using it, and another tries to swap out the
PCM for something new.
The PCMCache is a new type called MemoryBufferCache, which saves memory
buffers based on their filename. Its ownership is shared by the
CompilerInstance and ModuleManager.
- The ModuleManager stores PCMs there that it loads from disk, never
touching the disk if the cache is hot.
- When modules fail to validate, they're removed from the cache.
- When a CompilerInstance is spawned to build a new module, each
already-loaded PCM is assumed to be valid, and is frozen to avoid
the use-after-free.
- Any newly-built module is written directly to the cache to avoid the
round-trip to the filesystem, making lock files unnecessary for
correctness.
Original patch by Manman Ren; most testcases by Adrian Prantl!
llvm-svn: 298278
2017-03-21 01:58:26 +08:00
|
|
|
MemoryBufferCache PCMCache;
|
2017-01-06 09:04:46 +08:00
|
|
|
HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr,
|
|
|
|
Diags, OpenCLLangOpts, Target.get());
|
2013-10-12 17:29:48 +08:00
|
|
|
|
2017-01-06 03:11:36 +08:00
|
|
|
Preprocessor PP(std::make_shared<PreprocessorOptions>(), Diags,
|
Reapply "Modules: Cache PCMs in memory and avoid a use-after-free"
This reverts commit r298185, effectively reapplying r298165, after fixing the
new unit tests (PR32338). The memory buffer generator doesn't null-terminate
the MemoryBuffer it creates; this version of the commit informs getMemBuffer
about that to avoid the assert.
Original commit message follows:
----
Clang's internal build system for implicit modules uses lock files to
ensure that after a process writes a PCM it will read the same one back
in (without contention from other -cc1 commands). Since PCMs are read
from disk repeatedly while invalidating, building, and importing, the
lock is not released quickly. Furthermore, the LockFileManager is not
robust in every environment. Other -cc1 commands can stall until
timeout (after about eight minutes).
This commit changes the lock file from being necessary for correctness
to a (possibly dubious) performance hack. The remaining benefit is to
reduce duplicate work in competing -cc1 commands which depend on the
same module. Follow-up commits will change the internal build system to
continue after a timeout, and reduce the timeout. Perhaps we should
reconsider blocking at all.
This also fixes a use-after-free, when one part of a compilation
validates a PCM and starts using it, and another tries to swap out the
PCM for something new.
The PCMCache is a new type called MemoryBufferCache, which saves memory
buffers based on their filename. Its ownership is shared by the
CompilerInstance and ModuleManager.
- The ModuleManager stores PCMs there that it loads from disk, never
touching the disk if the cache is hot.
- When modules fail to validate, they're removed from the cache.
- When a CompilerInstance is spawned to build a new module, each
already-loaded PCM is assumed to be valid, and is frozen to avoid
the use-after-free.
- Any newly-built module is written directly to the cache to avoid the
round-trip to the filesystem, making lock files unnecessary for
correctness.
Original patch by Manman Ren; most testcases by Adrian Prantl!
llvm-svn: 298278
2017-03-21 01:58:26 +08:00
|
|
|
OpenCLLangOpts, SourceMgr, PCMCache, HeaderInfo, ModLoader,
|
2017-01-06 03:11:36 +08:00
|
|
|
/*IILookup =*/nullptr,
|
2014-05-02 11:43:30 +08:00
|
|
|
/*OwnsHeaderSearch =*/false);
|
|
|
|
PP.Initialize(*Target);
|
2013-10-12 17:29:48 +08:00
|
|
|
|
|
|
|
// parser actually sets correct pragma handlers for preprocessor
|
|
|
|
// according to LangOptions, so we init Parser to register opencl
|
|
|
|
// pragma handlers
|
2018-10-10 21:27:25 +08:00
|
|
|
ASTContext Context(OpenCLLangOpts, SourceMgr, PP.getIdentifierTable(),
|
|
|
|
PP.getSelectorTable(), PP.getBuiltinInfo());
|
2014-05-03 11:46:04 +08:00
|
|
|
Context.InitBuiltinTypes(*Target);
|
|
|
|
|
2013-10-12 17:29:48 +08:00
|
|
|
ASTConsumer Consumer;
|
|
|
|
Sema S(PP, Context, Consumer);
|
|
|
|
Parser P(PP, S, false);
|
|
|
|
PragmaOpenCLExtensionCallbacks* Callbacks = new PragmaOpenCLExtensionCallbacks;
|
2014-09-10 12:53:53 +08:00
|
|
|
PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(Callbacks));
|
2013-10-12 17:29:48 +08:00
|
|
|
|
|
|
|
// Lex source text.
|
|
|
|
PP.EnterMainSourceFile();
|
|
|
|
while (true) {
|
|
|
|
Token Tok;
|
|
|
|
PP.Lex(Tok);
|
|
|
|
if (Tok.is(tok::eof))
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
PragmaOpenCLExtensionCallbacks::CallbackParameters RetVal = {
|
2013-10-14 15:13:59 +08:00
|
|
|
Callbacks->Name,
|
2013-10-12 17:29:48 +08:00
|
|
|
Callbacks->State
|
|
|
|
};
|
2018-10-10 21:27:25 +08:00
|
|
|
return RetVal;
|
2013-10-12 17:29:48 +08:00
|
|
|
}
|
2012-11-02 01:52:58 +08:00
|
|
|
};
|
|
|
|
|
2018-05-11 03:05:36 +08:00
|
|
|
TEST_F(PPCallbacksTest, UserFileCharacteristics) {
|
|
|
|
const char *Source = "#include \"quoted.h\"\n";
|
|
|
|
|
|
|
|
SrcMgr::CharacteristicKind Kind =
|
|
|
|
InclusionDirectiveCharacteristicKind(Source, "/quoted.h", false);
|
|
|
|
|
|
|
|
ASSERT_EQ(SrcMgr::CharacteristicKind::C_User, Kind);
|
|
|
|
}
|
|
|
|
|
2012-11-02 01:52:58 +08:00
|
|
|
TEST_F(PPCallbacksTest, QuotedFilename) {
|
|
|
|
const char* Source =
|
|
|
|
"#include \"quoted.h\"\n";
|
|
|
|
|
|
|
|
CharSourceRange Range =
|
|
|
|
InclusionDirectiveFilenameRange(Source, "/quoted.h", false);
|
|
|
|
|
|
|
|
ASSERT_EQ("\"quoted.h\"", GetSourceString(Range));
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST_F(PPCallbacksTest, AngledFilename) {
|
|
|
|
const char* Source =
|
|
|
|
"#include <angled.h>\n";
|
|
|
|
|
|
|
|
CharSourceRange Range =
|
|
|
|
InclusionDirectiveFilenameRange(Source, "/angled.h", true);
|
|
|
|
|
|
|
|
ASSERT_EQ("<angled.h>", GetSourceString(Range));
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST_F(PPCallbacksTest, QuotedInMacro) {
|
|
|
|
const char* Source =
|
|
|
|
"#define MACRO_QUOTED \"quoted.h\"\n"
|
|
|
|
"#include MACRO_QUOTED\n";
|
|
|
|
|
|
|
|
CharSourceRange Range =
|
|
|
|
InclusionDirectiveFilenameRange(Source, "/quoted.h", false);
|
|
|
|
|
|
|
|
ASSERT_EQ("\"quoted.h\"", GetSourceString(Range));
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST_F(PPCallbacksTest, AngledInMacro) {
|
|
|
|
const char* Source =
|
|
|
|
"#define MACRO_ANGLED <angled.h>\n"
|
|
|
|
"#include MACRO_ANGLED\n";
|
|
|
|
|
|
|
|
CharSourceRange Range =
|
|
|
|
InclusionDirectiveFilenameRange(Source, "/angled.h", true);
|
|
|
|
|
|
|
|
ASSERT_EQ("<angled.h>", GetSourceString(Range));
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST_F(PPCallbacksTest, StringizedMacroArgument) {
|
|
|
|
const char* Source =
|
|
|
|
"#define MACRO_STRINGIZED(x) #x\n"
|
|
|
|
"#include MACRO_STRINGIZED(quoted.h)\n";
|
|
|
|
|
|
|
|
CharSourceRange Range =
|
|
|
|
InclusionDirectiveFilenameRange(Source, "/quoted.h", false);
|
|
|
|
|
|
|
|
ASSERT_EQ("\"quoted.h\"", GetSourceString(Range));
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST_F(PPCallbacksTest, ConcatenatedMacroArgument) {
|
|
|
|
const char* Source =
|
|
|
|
"#define MACRO_ANGLED <angled.h>\n"
|
|
|
|
"#define MACRO_CONCAT(x, y) x ## _ ## y\n"
|
|
|
|
"#include MACRO_CONCAT(MACRO, ANGLED)\n";
|
|
|
|
|
|
|
|
CharSourceRange Range =
|
|
|
|
InclusionDirectiveFilenameRange(Source, "/angled.h", false);
|
|
|
|
|
|
|
|
ASSERT_EQ("<angled.h>", GetSourceString(Range));
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST_F(PPCallbacksTest, TrigraphFilename) {
|
|
|
|
const char* Source =
|
2012-11-04 04:58:26 +08:00
|
|
|
"#include \"tri\?\?-graph.h\"\n";
|
2012-11-02 01:52:58 +08:00
|
|
|
|
|
|
|
CharSourceRange Range =
|
|
|
|
InclusionDirectiveFilenameRange(Source, "/tri~graph.h", false);
|
|
|
|
|
2012-11-04 04:58:26 +08:00
|
|
|
ASSERT_EQ("\"tri\?\?-graph.h\"", GetSourceString(Range));
|
2012-11-02 01:52:58 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST_F(PPCallbacksTest, TrigraphInMacro) {
|
|
|
|
const char* Source =
|
2012-11-04 04:58:26 +08:00
|
|
|
"#define MACRO_TRIGRAPH \"tri\?\?-graph.h\"\n"
|
2012-11-02 01:52:58 +08:00
|
|
|
"#include MACRO_TRIGRAPH\n";
|
|
|
|
|
|
|
|
CharSourceRange Range =
|
|
|
|
InclusionDirectiveFilenameRange(Source, "/tri~graph.h", false);
|
|
|
|
|
2012-11-04 04:58:26 +08:00
|
|
|
ASSERT_EQ("\"tri\?\?-graph.h\"", GetSourceString(Range));
|
2012-11-02 01:52:58 +08:00
|
|
|
}
|
|
|
|
|
2013-10-12 17:29:48 +08:00
|
|
|
TEST_F(PPCallbacksTest, OpenCLExtensionPragmaEnabled) {
|
|
|
|
const char* Source =
|
|
|
|
"#pragma OPENCL EXTENSION cl_khr_fp64 : enable\n";
|
|
|
|
|
|
|
|
PragmaOpenCLExtensionCallbacks::CallbackParameters Parameters =
|
|
|
|
PragmaOpenCLExtensionCall(Source);
|
|
|
|
|
|
|
|
ASSERT_EQ("cl_khr_fp64", Parameters.Name);
|
|
|
|
unsigned ExpectedState = 1;
|
|
|
|
ASSERT_EQ(ExpectedState, Parameters.State);
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST_F(PPCallbacksTest, OpenCLExtensionPragmaDisabled) {
|
|
|
|
const char* Source =
|
|
|
|
"#pragma OPENCL EXTENSION cl_khr_fp16 : disable\n";
|
|
|
|
|
|
|
|
PragmaOpenCLExtensionCallbacks::CallbackParameters Parameters =
|
|
|
|
PragmaOpenCLExtensionCall(Source);
|
|
|
|
|
|
|
|
ASSERT_EQ("cl_khr_fp16", Parameters.Name);
|
|
|
|
unsigned ExpectedState = 0;
|
|
|
|
ASSERT_EQ(ExpectedState, Parameters.State);
|
|
|
|
}
|
|
|
|
|
2019-01-11 05:22:13 +08:00
|
|
|
TEST_F(PPCallbacksTest, DirectiveExprRanges) {
|
2019-01-18 04:21:34 +08:00
|
|
|
const auto &Results1 = DirectiveExprRange("#if FLUZZY_FLOOF\n#endif\n");
|
|
|
|
EXPECT_EQ(Results1.size(), 1);
|
|
|
|
EXPECT_EQ(
|
|
|
|
GetSourceStringToEnd(CharSourceRange(Results1[0].ConditionRange, false)),
|
|
|
|
"FLUZZY_FLOOF");
|
|
|
|
|
|
|
|
const auto &Results2 = DirectiveExprRange("#if 1 + 4 < 7\n#endif\n");
|
|
|
|
EXPECT_EQ(Results2.size(), 1);
|
|
|
|
EXPECT_EQ(
|
|
|
|
GetSourceStringToEnd(CharSourceRange(Results2[0].ConditionRange, false)),
|
|
|
|
"1 + 4 < 7");
|
|
|
|
|
|
|
|
const auto &Results3 = DirectiveExprRange("#if 1 + \\\n 2\n#endif\n");
|
|
|
|
EXPECT_EQ(Results3.size(), 1);
|
|
|
|
EXPECT_EQ(
|
|
|
|
GetSourceStringToEnd(CharSourceRange(Results3[0].ConditionRange, false)),
|
|
|
|
"1 + \\\n 2");
|
|
|
|
|
|
|
|
const auto &Results4 = DirectiveExprRange("#if 0\n#elif FLOOFY\n#endif\n");
|
|
|
|
EXPECT_EQ(Results4.size(), 2);
|
|
|
|
EXPECT_EQ(
|
|
|
|
GetSourceStringToEnd(CharSourceRange(Results4[0].ConditionRange, false)),
|
|
|
|
"0");
|
|
|
|
EXPECT_EQ(
|
|
|
|
GetSourceStringToEnd(CharSourceRange(Results4[1].ConditionRange, false)),
|
|
|
|
"FLOOFY");
|
|
|
|
|
|
|
|
const auto &Results5 = DirectiveExprRange("#if 1\n#elif FLOOFY\n#endif\n");
|
|
|
|
EXPECT_EQ(Results5.size(), 2);
|
|
|
|
EXPECT_EQ(
|
|
|
|
GetSourceStringToEnd(CharSourceRange(Results5[0].ConditionRange, false)),
|
|
|
|
"1");
|
|
|
|
EXPECT_EQ(
|
|
|
|
GetSourceStringToEnd(CharSourceRange(Results5[1].ConditionRange, false)),
|
|
|
|
"FLOOFY");
|
|
|
|
|
|
|
|
const auto &Results6 =
|
|
|
|
DirectiveExprRange("#if defined(FLUZZY_FLOOF)\n#endif\n");
|
|
|
|
EXPECT_EQ(Results6.size(), 1);
|
|
|
|
EXPECT_EQ(
|
|
|
|
GetSourceStringToEnd(CharSourceRange(Results6[0].ConditionRange, false)),
|
|
|
|
"defined(FLUZZY_FLOOF)");
|
|
|
|
|
|
|
|
const auto &Results7 =
|
|
|
|
DirectiveExprRange("#if 1\n#elif defined(FLOOFY)\n#endif\n");
|
|
|
|
EXPECT_EQ(Results7.size(), 2);
|
|
|
|
EXPECT_EQ(
|
|
|
|
GetSourceStringToEnd(CharSourceRange(Results7[0].ConditionRange, false)),
|
|
|
|
"1");
|
|
|
|
EXPECT_EQ(
|
|
|
|
GetSourceStringToEnd(CharSourceRange(Results7[1].ConditionRange, false)),
|
|
|
|
"defined(FLOOFY)");
|
|
|
|
|
2019-01-16 01:20:05 +08:00
|
|
|
const auto &Results8 =
|
|
|
|
DirectiveExprRange("#define FLOOFY 0\n#if __FILE__ > FLOOFY\n#endif\n");
|
|
|
|
EXPECT_EQ(Results8.size(), 1U);
|
2019-01-11 05:22:13 +08:00
|
|
|
EXPECT_EQ(
|
2019-01-16 01:20:05 +08:00
|
|
|
GetSourceStringToEnd(CharSourceRange(Results8[0].ConditionRange, false)),
|
2019-01-18 04:21:34 +08:00
|
|
|
"__FILE__ > FLOOFY");
|
2019-01-11 05:22:13 +08:00
|
|
|
EXPECT_EQ(
|
2019-01-16 01:20:05 +08:00
|
|
|
Lexer::getSourceText(CharSourceRange(Results8[0].ConditionRange, false),
|
|
|
|
SourceMgr, LangOpts),
|
2019-01-18 04:21:34 +08:00
|
|
|
"__FILE__ > FLOOFY");
|
2019-01-11 05:22:13 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
} // namespace
|