PR9992: Serialize and deserialize the token sequence for a function template in

-fdelayed-template-parsing mode. Patch by Will Wilson!

llvm-svn: 187916
This commit is contained in:
Richard Smith 2013-08-07 21:41:30 +00:00
parent 2c1a894061
commit e40f2baa5d
19 changed files with 179 additions and 83 deletions

View File

@ -1051,26 +1051,10 @@ private:
SourceRange getSourceRange() const LLVM_READONLY;
};
/// \brief Contains a late templated function.
/// Will be parsed at the end of the translation unit.
struct LateParsedTemplatedFunction {
explicit LateParsedTemplatedFunction(Decl *MD)
: D(MD) {}
CachedTokens Toks;
/// \brief The template function declaration to be late parsed.
Decl *D;
};
void LexTemplateFunctionForLateParsing(CachedTokens &Toks);
void ParseLateTemplatedFuncDef(LateParsedTemplatedFunction &LMT);
typedef llvm::DenseMap<const FunctionDecl*, LateParsedTemplatedFunction*>
LateParsedTemplateMapT;
LateParsedTemplateMapT LateParsedTemplateMap;
void ParseLateTemplatedFuncDef(LateParsedTemplate &LPT);
static void LateTemplateParserCallback(void *P, const FunctionDecl *FD);
void LateTemplateParser(const FunctionDecl *FD);
static void LateTemplateParserCallback(void *P, LateParsedTemplate &LPT);
Sema::ParsingClassState
PushParsingClass(Decl *TagOrTemplate, bool TopLevelClass, bool IsInterface);

View File

@ -30,7 +30,8 @@ class Sema;
class TypedefNameDecl;
class ValueDecl;
class VarDecl;
struct LateParsedTemplate;
/// \brief A simple structure that captures a vtable use for the purposes of
/// the \c ExternalSemaSource.
struct ExternalVTableUse {
@ -177,6 +178,15 @@ public:
SmallVectorImpl<std::pair<ValueDecl *,
SourceLocation> > &Pending) {}
/// \brief Read the set of late parsed template functions for this source.
///
/// The external source should insert its own late parsed template functions
/// into the map. Note that this routine may be invoked multiple times; the
/// external source should take care not to introduce the same map entries
/// repeatedly.
virtual void ReadLateParsedTemplates(
llvm::DenseMap<const FunctionDecl *, LateParsedTemplate *> &LPTMap) {}
// isa/cast/dyn_cast support
static bool classof(const ExternalASTSource *Source) {
return Source->SemaSource;

View File

@ -322,6 +322,15 @@ public:
virtual void ReadPendingInstantiations(
SmallVectorImpl<std::pair<ValueDecl*, SourceLocation> >& Pending);
/// \brief Read the set of late parsed template functions for this source.
///
/// The external source should insert its own late parsed template functions
/// into the map. Note that this routine may be invoked multiple times; the
/// external source should take care not to introduce the same map entries
/// repeatedly.
virtual void ReadLateParsedTemplates(
llvm::DenseMap<const FunctionDecl *, LateParsedTemplate *> &LPTMap);
// isa/cast/dyn_cast support
static bool classof(const MultiplexExternalSemaSource*) { return true; }
//static bool classof(const ExternalSemaSource*) { return true; }

View File

@ -394,8 +394,12 @@ public:
SmallVector<std::pair<CXXMethodDecl*, const FunctionProtoType*>, 2>
DelayedDefaultedMemberExceptionSpecs;
typedef llvm::DenseMap<const FunctionDecl *, LateParsedTemplate *>
LateParsedTemplateMapT;
LateParsedTemplateMapT LateParsedTemplateMap;
/// \brief Callback to the parser to parse templated functions when needed.
typedef void LateTemplateParserCB(void *P, const FunctionDecl *FD);
typedef void LateTemplateParserCB(void *P, LateParsedTemplate &LPT);
LateTemplateParserCB *LateTemplateParser;
void *OpaqueParser;
@ -4688,7 +4692,9 @@ public:
void ActOnFinishDelayedMemberDeclarations(Scope *S, Decl *Record);
void ActOnFinishDelayedCXXMethodDeclaration(Scope *S, Decl *Method);
void ActOnFinishDelayedMemberInitializers(Decl *Record);
void MarkAsLateParsedTemplate(FunctionDecl *FD, bool Flag = true);
void MarkAsLateParsedTemplate(FunctionDecl *FD, Decl *FnD,
CachedTokens &Toks);
void UnmarkAsLateParsedTemplate(FunctionDecl *FD);
bool IsInsideALocalClassWithinATemplateFunction();
Decl *ActOnStaticAssertDeclaration(SourceLocation StaticAssertLoc,
@ -7829,6 +7835,14 @@ DeductionFailureInfo
MakeDeductionFailureInfo(ASTContext &Context, Sema::TemplateDeductionResult TDK,
sema::TemplateDeductionInfo &Info);
} // end namespace clang
/// \brief Contains a late templated function.
/// Will be parsed at the end of the translation unit, used by Sema & Parser.
struct LateParsedTemplate {
CachedTokens Toks;
/// \brief The template function declaration to be late parsed.
Decl *D;
};
} // end namespace clang
#endif

View File

@ -535,7 +535,10 @@ namespace clang {
/// \brief Record code for undefined but used functions and variables that
/// need a definition in this TU.
UNDEFINED_BUT_USED = 49
UNDEFINED_BUT_USED = 49,
/// \brief Record code for late parsed template functions.
LATE_PARSED_TEMPLATE = 50
};
/// \brief Record types used within a source manager block.

View File

@ -240,6 +240,7 @@ class ASTReader
{
public:
typedef SmallVector<uint64_t, 64> RecordData;
typedef SmallVectorImpl<uint64_t> RecordDataImpl;
/// \brief The result of reading the control block of an AST file, which
/// can fail for various reasons.
@ -710,6 +711,9 @@ private:
/// SourceLocation of a matching ODR-use.
SmallVector<uint64_t, 8> UndefinedButUsed;
// \brief A list of late parsed template function data.
SmallVector<uint64_t, 1> LateParsedTemplates;
/// \brief A list of modules that were imported by precompiled headers or
/// any other non-module AST file.
SmallVector<serialization::SubmoduleID, 2> ImportedModules;
@ -1607,6 +1611,9 @@ public:
SmallVectorImpl<std::pair<ValueDecl *,
SourceLocation> > &Pending);
virtual void ReadLateParsedTemplates(
llvm::DenseMap<const FunctionDecl *, LateParsedTemplate *> &LPTMap);
/// \brief Load a selector from disk, registering its ID if it exists.
void LoadSelector(Selector Sel);
@ -1757,7 +1764,8 @@ public:
/// \brief Read a source location.
SourceLocation ReadSourceLocation(ModuleFile &ModuleFile,
const RecordData &Record, unsigned &Idx) {
const RecordDataImpl &Record,
unsigned &Idx) {
return ReadSourceLocation(ModuleFile, Record[Idx++]);
}
@ -1808,7 +1816,7 @@ public:
Expr *ReadSubExpr();
/// \brief Reads a token out of a record.
Token ReadToken(ModuleFile &M, const RecordData &Record, unsigned &Idx);
Token ReadToken(ModuleFile &M, const RecordDataImpl &Record, unsigned &Idx);
/// \brief Reads the macro record located at the given offset.
MacroInfo *ReadMacroRecord(ModuleFile &F, uint64_t Offset);

View File

@ -457,7 +457,8 @@ private:
void WriteObjCCategories();
void WriteRedeclarations();
void WriteMergedDecls();
void WriteLateParsedTemplates(Sema &SemaRef);
unsigned DeclParmVarAbbrev;
unsigned DeclContextLexicalAbbrev;
unsigned DeclContextVisibleLookupAbbrev;

View File

@ -120,18 +120,13 @@ NamedDecl *Parser::ParseCXXInlineMethodDef(AccessSpecifier AS,
TemplateInfo.Kind != ParsedTemplateInfo::NonTemplate) &&
!Actions.IsInsideALocalClassWithinATemplateFunction())) {
if (FnD) {
LateParsedTemplatedFunction *LPT = new LateParsedTemplatedFunction(FnD);
CachedTokens Toks;
LexTemplateFunctionForLateParsing(Toks);
if (FnD) {
FunctionDecl *FD = getFunctionDecl(FnD);
Actions.CheckForFunctionRedefinition(FD);
LateParsedTemplateMap[FD] = LPT;
Actions.MarkAsLateParsedTemplate(FD);
LexTemplateFunctionForLateParsing(LPT->Toks);
} else {
CachedTokens Toks;
LexTemplateFunctionForLateParsing(Toks);
Actions.MarkAsLateParsedTemplate(FD, FnD, Toks);
}
return FnD;

View File

@ -1241,30 +1241,19 @@ SourceRange Parser::ParsedTemplateInfo::getSourceRange() const {
return R;
}
void Parser::LateTemplateParserCallback(void *P, const FunctionDecl *FD) {
((Parser*)P)->LateTemplateParser(FD);
}
void Parser::LateTemplateParser(const FunctionDecl *FD) {
LateParsedTemplatedFunction *LPT = LateParsedTemplateMap[FD];
if (LPT) {
ParseLateTemplatedFuncDef(*LPT);
return;
}
llvm_unreachable("Late templated function without associated lexed tokens");
void Parser::LateTemplateParserCallback(void *P, LateParsedTemplate &LPT) {
((Parser *)P)->ParseLateTemplatedFuncDef(LPT);
}
/// \brief Late parse a C++ function template in Microsoft mode.
void Parser::ParseLateTemplatedFuncDef(LateParsedTemplatedFunction &LMT) {
if(!LMT.D)
void Parser::ParseLateTemplatedFuncDef(LateParsedTemplate &LPT) {
if(!LPT.D)
return;
// Get the FunctionDecl.
FunctionTemplateDecl *FunTmplD = dyn_cast<FunctionTemplateDecl>(LMT.D);
FunctionTemplateDecl *FunTmplD = dyn_cast<FunctionTemplateDecl>(LPT.D);
FunctionDecl *FunD =
FunTmplD ? FunTmplD->getTemplatedDecl() : cast<FunctionDecl>(LMT.D);
FunTmplD ? FunTmplD->getTemplatedDecl() : cast<FunctionDecl>(LPT.D);
// Track template parameter depth.
TemplateParameterDepthRAII CurTemplateDepthTracker(TemplateParameterDepth);
@ -1312,15 +1301,15 @@ void Parser::ParseLateTemplatedFuncDef(LateParsedTemplatedFunction &LMT) {
Actions.ActOnReenterDeclaratorTemplateScope(getCurScope(), Declarator);
++CurTemplateDepthTracker;
}
Actions.ActOnReenterTemplateScope(getCurScope(), LMT.D);
Actions.ActOnReenterTemplateScope(getCurScope(), LPT.D);
++CurTemplateDepthTracker;
assert(!LMT.Toks.empty() && "Empty body!");
assert(!LPT.Toks.empty() && "Empty body!");
// Append the current token at the end of the new token stream so that it
// doesn't get lost.
LMT.Toks.push_back(Tok);
PP.EnterTokenStream(LMT.Toks.data(), LMT.Toks.size(), true, false);
LPT.Toks.push_back(Tok);
PP.EnterTokenStream(LPT.Toks.data(), LPT.Toks.size(), true, false);
// Consume the previously pushed token.
ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true);
@ -1337,22 +1326,22 @@ void Parser::ParseLateTemplatedFuncDef(LateParsedTemplatedFunction &LMT) {
Actions.ActOnStartOfFunctionDef(getCurScope(), FunD);
if (Tok.is(tok::kw_try)) {
ParseFunctionTryBlock(LMT.D, FnScope);
ParseFunctionTryBlock(LPT.D, FnScope);
} else {
if (Tok.is(tok::colon))
ParseConstructorInitializer(LMT.D);
ParseConstructorInitializer(LPT.D);
else
Actions.ActOnDefaultCtorInitializers(LMT.D);
Actions.ActOnDefaultCtorInitializers(LPT.D);
if (Tok.is(tok::l_brace)) {
assert((!FunTmplD || FunTmplD->getTemplateParameters()->getDepth() <
TemplateParameterDepth) &&
"TemplateParameterDepth should be greater than the depth of "
"current template being instantiated!");
ParseFunctionStatementBody(LMT.D, FnScope);
Actions.MarkAsLateParsedTemplate(FunD, false);
ParseFunctionStatementBody(LPT.D, FnScope);
Actions.UnmarkAsLateParsedTemplate(FunD);
} else
Actions.ActOnFinishFunctionBody(LMT.D, 0);
Actions.ActOnFinishFunctionBody(LPT.D, 0);
}
// Exit scopes.
@ -1362,7 +1351,7 @@ void Parser::ParseLateTemplatedFuncDef(LateParsedTemplatedFunction &LMT) {
for (; I != TemplateParamScopeStack.rend(); ++I)
delete *I;
DeclGroupPtrTy grp = Actions.ConvertDeclToDeclGroup(LMT.D);
DeclGroupPtrTy grp = Actions.ConvertDeclToDeclGroup(LPT.D);
if (grp)
Actions.getASTConsumer().HandleTopLevelDecl(grp.get());
}

View File

@ -422,11 +422,6 @@ Parser::~Parser() {
for (unsigned i = 0, e = NumCachedScopes; i != e; ++i)
delete ScopeCache[i];
// Free LateParsedTemplatedFunction nodes.
for (LateParsedTemplateMapT::iterator it = LateParsedTemplateMap.begin();
it != LateParsedTemplateMap.end(); ++it)
delete it->second;
// Remove the pragma handlers we installed.
PP.RemovePragmaHandler(AlignHandler.get());
AlignHandler.reset();
@ -1008,22 +1003,18 @@ Decl *Parser::ParseFunctionDefinition(ParsingDeclarator &D,
D.complete(DP);
D.getMutableDeclSpec().abort();
if (DP) {
LateParsedTemplatedFunction *LPT = new LateParsedTemplatedFunction(DP);
CachedTokens Toks;
LexTemplateFunctionForLateParsing(Toks);
if (DP) {
FunctionDecl *FnD = 0;
if (FunctionTemplateDecl *FunTmpl = dyn_cast<FunctionTemplateDecl>(DP))
FnD = FunTmpl->getTemplatedDecl();
else
FnD = cast<FunctionDecl>(DP);
Actions.CheckForFunctionRedefinition(FnD);
LateParsedTemplateMap[FnD] = LPT;
Actions.MarkAsLateParsedTemplate(FnD);
LexTemplateFunctionForLateParsing(LPT->Toks);
} else {
CachedTokens Toks;
LexTemplateFunctionForLateParsing(Toks);
Actions.CheckForFunctionRedefinition(FnD);
Actions.MarkAsLateParsedTemplate(FnD, DP, Toks);
}
return DP;
}

View File

@ -267,3 +267,9 @@ void MultiplexExternalSemaSource::ReadPendingInstantiations(
for(size_t i = 0; i < Sources.size(); ++i)
Sources[i]->ReadPendingInstantiations(Pending);
}
void MultiplexExternalSemaSource::ReadLateParsedTemplates(
llvm::DenseMap<const FunctionDecl *, LateParsedTemplate *> &LPTMap) {
for (size_t i = 0; i < Sources.size(); ++i)
Sources[i]->ReadLateParsedTemplates(LPTMap);
}

View File

@ -173,6 +173,10 @@ void Sema::Initialize() {
}
Sema::~Sema() {
for (LateParsedTemplateMapT::iterator I = LateParsedTemplateMap.begin(),
E = LateParsedTemplateMap.end();
I != E; ++I)
delete I->second;
if (PackContext) FreePackedContext();
if (VisContext) FreeVisContext();
delete TheTargetAttributesSema;

View File

@ -7902,11 +7902,26 @@ Sema::getTemplateArgumentBindingsText(const TemplateParameterList *Params,
return Out.str();
}
void Sema::MarkAsLateParsedTemplate(FunctionDecl *FD, bool Flag) {
void Sema::MarkAsLateParsedTemplate(FunctionDecl *FD, Decl *FnD,
CachedTokens &Toks) {
if (!FD)
return;
FD->setLateTemplateParsed(Flag);
}
LateParsedTemplate *LPT = new LateParsedTemplate;
// Take tokens to avoid allocations
LPT->Toks.swap(Toks);
LPT->D = FnD;
LateParsedTemplateMap[FD] = LPT;
FD->setLateTemplateParsed(true);
}
void Sema::UnmarkAsLateParsedTemplate(FunctionDecl *FD) {
if (!FD)
return;
FD->setLateTemplateParsed(false);
}
bool Sema::IsInsideALocalClassWithinATemplateFunction() {
DeclContext *DC = CurContext;

View File

@ -3112,7 +3112,13 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
// a templated function definition.
if (!Pattern && PatternDecl->isLateTemplateParsed() &&
LateTemplateParser) {
LateTemplateParser(OpaqueParser, PatternDecl);
// FIXME: Optimize to allow individual templates to be deserialized.
if (PatternDecl->isFromASTFile())
ExternalSource->ReadLateParsedTemplates(LateParsedTemplateMap);
LateParsedTemplate *LPT = LateParsedTemplateMap.lookup(PatternDecl);
assert(LPT && "missing LateParsedTemplate");
LateTemplateParser(OpaqueParser, *LPT);
Pattern = PatternDecl->getBody(PatternDecl);
}

View File

@ -1103,7 +1103,7 @@ bool ASTReader::ReadBlockAbbrevs(BitstreamCursor &Cursor, unsigned BlockID) {
}
}
Token ASTReader::ReadToken(ModuleFile &F, const RecordData &Record,
Token ASTReader::ReadToken(ModuleFile &F, const RecordDataImpl &Record,
unsigned &Idx) {
Token Tok;
Tok.startToken();
@ -2751,6 +2751,11 @@ bool ASTReader::ReadASTBlock(ModuleFile &F) {
// FIXME: Not used yet.
break;
}
case LATE_PARSED_TEMPLATE: {
LateParsedTemplates.append(Record.begin(), Record.end());
break;
}
}
}
}
@ -6475,6 +6480,29 @@ void ASTReader::ReadPendingInstantiations(
PendingInstantiations.clear();
}
void ASTReader::ReadLateParsedTemplates(
llvm::DenseMap<const FunctionDecl *, LateParsedTemplate *> &LPTMap) {
for (unsigned Idx = 0, N = LateParsedTemplates.size(); Idx < N;
/* In loop */) {
FunctionDecl *FD = cast<FunctionDecl>(GetDecl(LateParsedTemplates[Idx++]));
LateParsedTemplate *LT = new LateParsedTemplate;
LT->D = GetDecl(LateParsedTemplates[Idx++]);
ModuleFile *F = getOwningModuleFile(LT->D);
assert(F && "No module");
unsigned TokN = LateParsedTemplates[Idx++];
LT->Toks.reserve(TokN);
for (unsigned T = 0; T < TokN; ++T)
LT->Toks.push_back(ReadToken(*F, LateParsedTemplates, Idx));
LPTMap[FD] = LT;
}
LateParsedTemplates.clear();
}
void ASTReader::LoadSelector(Selector Sel) {
// It would be complicated to avoid reading the methods anyway. So don't.
ReadMethodPool(Sel);

View File

@ -544,6 +544,7 @@ void ASTDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
FD->HasImplicitReturnZero = Record[Idx++];
FD->IsConstexpr = Record[Idx++];
FD->HasSkippedBody = Record[Idx++];
FD->IsLateTemplateParsed = Record[Idx++];
FD->setCachedLinkage(Linkage(Record[Idx++]));
FD->EndRangeLoc = ReadSourceLocation(Record, Idx);

View File

@ -848,6 +848,7 @@ void ASTWriter::WriteBlockInfoBlock() {
RECORD(OBJC_CATEGORIES);
RECORD(MACRO_OFFSET);
RECORD(MACRO_TABLE);
RECORD(LATE_PARSED_TEMPLATE);
// SourceManager Block.
BLOCK(SOURCE_MANAGER_BLOCK);
@ -3673,6 +3674,30 @@ void ASTWriter::WriteMergedDecls() {
Stream.EmitRecord(MERGED_DECLARATIONS, Record);
}
void ASTWriter::WriteLateParsedTemplates(Sema &SemaRef) {
Sema::LateParsedTemplateMapT &LPTMap = SemaRef.LateParsedTemplateMap;
if (LPTMap.empty())
return;
RecordData Record;
for (Sema::LateParsedTemplateMapT::iterator It = LPTMap.begin(),
ItEnd = LPTMap.end();
It != ItEnd; ++It) {
LateParsedTemplate *LPT = It->second;
AddDeclRef(It->first, Record);
AddDeclRef(LPT->D, Record);
Record.push_back(LPT->Toks.size());
for (CachedTokens::iterator TokIt = LPT->Toks.begin(),
TokEnd = LPT->Toks.end();
TokIt != TokEnd; ++TokIt) {
AddToken(*TokIt, Record);
}
}
Stream.EmitRecord(LATE_PARSED_TEMPLATE, Record);
}
//===----------------------------------------------------------------------===//
// General Serialization Routines
//===----------------------------------------------------------------------===//
@ -4255,7 +4280,8 @@ void ASTWriter::WriteASTCore(Sema &SemaRef,
WriteRedeclarations();
WriteMergedDecls();
WriteObjCCategories();
WriteLateParsedTemplates(SemaRef);
// Some simple statistics
Record.clear();
Record.push_back(NumStatements);

View File

@ -342,6 +342,7 @@ void ASTDeclWriter::VisitFunctionDecl(FunctionDecl *D) {
Record.push_back(D->hasImplicitReturnZero());
Record.push_back(D->isConstexpr());
Record.push_back(D->HasSkippedBody);
Record.push_back(D->isLateTemplateParsed());
Record.push_back(D->getLinkageInternal());
Writer.AddSourceLocation(D->getLocEnd(), Record);

View File

@ -12,6 +12,11 @@
// RUN: %clang_cc1 -std=c++11 -fcxx-exceptions -fexceptions -fmodules -include-pch %t -verify %s -ast-dump -o -
// RUN: %clang_cc1 -std=c++11 -fcxx-exceptions -fexceptions -fmodules -include-pch %t %s -emit-llvm -o - -error-on-deserialized-decl doNotDeserialize | FileCheck %s
// Test with pch and delayed template parsing.
// RUN: %clang_cc1 -std=c++11 -fcxx-exceptions -fdelayed-template-parsing -fexceptions -x c++-header -emit-pch -o %t %S/cxx-templates.h
// RUN: %clang_cc1 -std=c++11 -fcxx-exceptions -fdelayed-template-parsing -fexceptions -include-pch %t -verify %s -ast-dump -o -
// RUN: %clang_cc1 -std=c++11 -fcxx-exceptions -fdelayed-template-parsing -fexceptions -include-pch %t %s -emit-llvm -o - | FileCheck %s
// expected-no-diagnostics
// CHECK: define weak_odr void @_ZN2S4IiE1mEv