diff --git a/clang/include/clang/Basic/IdentifierTable.h b/clang/include/clang/Basic/IdentifierTable.h index a06e9b10807b..56a6f4184852 100644 --- a/clang/include/clang/Basic/IdentifierTable.h +++ b/clang/include/clang/Basic/IdentifierTable.h @@ -122,14 +122,7 @@ public: /// can be used to cause the lexer to map identifiers to source-language /// tokens. tok::TokenKind getTokenID() const { return (tok::TokenKind)TokenID; } - void setTokenID(tok::TokenKind ID) { - TokenID = ID; - - if (ID != tok::identifier) - NeedsHandleIdentifier = 1; - else - RecomputeNeedsHandleIdentifier(); - } + void setTokenID(tok::TokenKind ID) { TokenID = ID; } /// getPPKeywordID - Return the preprocessor keyword ID for this identifier. /// For example, "define" will return tok::pp_define. @@ -225,7 +218,7 @@ private: void RecomputeNeedsHandleIdentifier() { NeedsHandleIdentifier = (isPoisoned() | hasMacroDefinition() | isCPlusPlusOperatorKeyword() | - isExtensionToken()) || getTokenID() != tok::identifier; + isExtensionToken()); } }; diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp index fca8fd4eb165..3174a0591745 100644 --- a/clang/lib/Lex/Lexer.cpp +++ b/clang/lib/Lex/Lexer.cpp @@ -558,6 +558,10 @@ FinishIdentifier: // identifier table. IdentifierInfo *II = PP->LookUpIdentifierInfo(Result, IdStart); + // Change the kind of this identifier to the appropriate token kind, e.g. + // turning "for" into a keyword. + Result.setKind(II->getTokenID()); + // Finally, now that we know we have an identifier, pass this off to the // preprocessor, which may macro expand it or something. if (II->isHandleIdentifierCase()) diff --git a/clang/lib/Lex/PTHLexer.cpp b/clang/lib/Lex/PTHLexer.cpp index b0f06271c4fb..ec76a299845b 100644 --- a/clang/lib/Lex/PTHLexer.cpp +++ b/clang/lib/Lex/PTHLexer.cpp @@ -101,7 +101,13 @@ LexNextToken: if (IdentifierID) { MIOpt.ReadToken(); IdentifierInfo *II = PTHMgr.GetIdentifierInfo(IdentifierID-1); + Tok.setIdentifierInfo(II); + + // Change the kind of this identifier to the appropriate token kind, e.g. + // turning "for" into a keyword. + Tok.setKind(II->getTokenID()); + if (II->isHandleIdentifierCase()) PP->HandleIdentifier(Tok); return; diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp index e53c392c388b..d0a15e45c414 100644 --- a/clang/lib/Lex/Preprocessor.cpp +++ b/clang/lib/Lex/Preprocessor.cpp @@ -759,10 +759,6 @@ void Preprocessor::HandleIdentifier(Token &Identifier) { if (II.isCPlusPlusOperatorKeyword()) Identifier.setIdentifierInfo(0); - // Change the kind of this identifier to the appropriate token kind, e.g. - // turning "for" into a keyword. - Identifier.setKind(II.getTokenID()); - // If this is an extension token, diagnose its use. // We avoid diagnosing tokens that originate from macro definitions. if (II.isExtensionToken() && Features.C99 && !DisableMacroExpansion) diff --git a/clang/lib/Lex/TokenLexer.cpp b/clang/lib/Lex/TokenLexer.cpp index c945843459f4..dd5352c1b616 100644 --- a/clang/lib/Lex/TokenLexer.cpp +++ b/clang/lib/Lex/TokenLexer.cpp @@ -326,9 +326,14 @@ void TokenLexer::Lex(Token &Tok) { } // Handle recursive expansion! - if (Tok.getIdentifierInfo() && !DisableMacroExpansion && - Tok.getIdentifierInfo()->isHandleIdentifierCase()) - PP.HandleIdentifier(Tok); + if (IdentifierInfo *II = Tok.getIdentifierInfo()) { + // Change the kind of this identifier to the appropriate token kind, e.g. + // turning "for" into a keyword. + Tok.setKind(II->getTokenID()); + + if (!DisableMacroExpansion && II->isHandleIdentifierCase()) + PP.HandleIdentifier(Tok); + } // Otherwise, return a normal token. }