diff --git a/clang/include/clang/Basic/IdentifierTable.h b/clang/include/clang/Basic/IdentifierTable.h index bd0b539d2094..97cd68ee5513 100644 --- a/clang/include/clang/Basic/IdentifierTable.h +++ b/clang/include/clang/Basic/IdentifierTable.h @@ -252,7 +252,7 @@ private: void RecomputeNeedsHandleIdentifier() { NeedsHandleIdentifier = (isPoisoned() | hasMacroDefinition() | isCPlusPlusOperatorKeyword() | - isExtensionToken()); + isExtensionToken() || (getTokenID() == tok::kw___import_module__)); } }; diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h index 47af19e49661..c404149f1892 100644 --- a/clang/include/clang/Lex/Preprocessor.h +++ b/clang/include/clang/Lex/Preprocessor.h @@ -119,9 +119,6 @@ class Preprocessor : public llvm::RefCountedBase { /// \brief Whether we have already loaded macros from the external source. mutable bool ReadMacrosFromExternalSource : 1; - /// \brief Tracks the depth of Lex() Calls. - unsigned LexDepth; - /// Identifiers - This is mapping/lookup information for all identifiers in /// the program, including program keywords. mutable IdentifierTable Identifiers; @@ -164,6 +161,10 @@ class Preprocessor : public llvm::RefCountedBase { /// for preprocessing. SourceLocation CodeCompletionFileLoc; + /// \brief The source location of the __import_module__ keyword we just + /// lexed, if any. + SourceLocation ModuleImportLoc; + /// \brief True if we hit the code-completion point. bool CodeCompletionReached; @@ -187,7 +188,7 @@ class Preprocessor : public llvm::RefCountedBase { /// if not expanding a macro. This is an alias for either CurLexer or /// CurPTHLexer. PreprocessorLexer *CurPPLexer; - + /// CurLookup - The DirectoryLookup structure used to find the current /// FileEntry, if CurLexer is non-null and if applicable. This allows us to /// implement #include_next and find directory-specific properties. @@ -197,20 +198,31 @@ class Preprocessor : public llvm::RefCountedBase { /// expanding a macro. One of CurLexer and CurTokenLexer must be null. llvm::OwningPtr CurTokenLexer; + /// \brief The kind of lexer we're currently working with. + enum CurLexerKind { + CLK_Lexer, + CLK_PTHLexer, + CLK_TokenLexer, + CLK_CachingLexer, + CLK_LexAfterModuleImport + } CurLexerKind; + /// IncludeMacroStack - This keeps track of the stack of files currently /// #included, and macros currently being expanded from, not counting /// CurLexer/CurTokenLexer. struct IncludeStackInfo { + enum CurLexerKind CurLexerKind; Lexer *TheLexer; PTHLexer *ThePTHLexer; PreprocessorLexer *ThePPLexer; TokenLexer *TheTokenLexer; const DirectoryLookup *TheDirLookup; - IncludeStackInfo(Lexer *L, PTHLexer* P, PreprocessorLexer* PPL, + IncludeStackInfo(enum CurLexerKind K, Lexer *L, PTHLexer* P, + PreprocessorLexer* PPL, TokenLexer* TL, const DirectoryLookup *D) - : TheLexer(L), ThePTHLexer(P), ThePPLexer(PPL), TheTokenLexer(TL), - TheDirLookup(D) {} + : CurLexerKind(K), TheLexer(L), ThePTHLexer(P), ThePPLexer(PPL), + TheTokenLexer(TL), TheDirLookup(D) {} }; std::vector IncludeMacroStack; @@ -555,22 +567,17 @@ public: /// Lex - To lex a token from the preprocessor, just pull a token from the /// current lexer or macro object. void Lex(Token &Result) { - ++LexDepth; - if (CurLexer) - CurLexer->Lex(Result); - else if (CurPTHLexer) - CurPTHLexer->Lex(Result); - else if (CurTokenLexer) - CurTokenLexer->Lex(Result); - else - CachingLex(Result); - --LexDepth; - - // If we have the __import_module__ keyword, handle the module import now. - if (Result.getKind() == tok::kw___import_module__ && LexDepth == 0) - HandleModuleImport(Result); + switch (CurLexerKind) { + case CLK_Lexer: CurLexer->Lex(Result); break; + case CLK_PTHLexer: CurPTHLexer->Lex(Result); break; + case CLK_TokenLexer: CurTokenLexer->Lex(Result); break; + case CLK_CachingLexer: CachingLex(Result); break; + case CLK_LexAfterModuleImport: LexAfterModuleImport(Result); break; + } } + void LexAfterModuleImport(Token &Result); + /// LexNonComment - Lex a token. If it's a comment, keep lexing until we get /// something not a comment. This is useful in -E -C mode where comments /// would foul up preprocessor directive handling. @@ -997,7 +1004,8 @@ public: private: void PushIncludeMacroStack() { - IncludeMacroStack.push_back(IncludeStackInfo(CurLexer.take(), + IncludeMacroStack.push_back(IncludeStackInfo(CurLexerKind, + CurLexer.take(), CurPTHLexer.take(), CurPPLexer, CurTokenLexer.take(), @@ -1011,6 +1019,7 @@ private: CurPPLexer = IncludeMacroStack.back().ThePPLexer; CurTokenLexer.reset(IncludeMacroStack.back().TheTokenLexer); CurDirLookup = IncludeMacroStack.back().TheDirLookup; + CurLexerKind = IncludeMacroStack.back().CurLexerKind; IncludeMacroStack.pop_back(); } @@ -1065,9 +1074,6 @@ private: /// the macro should not be expanded return true, otherwise return false. bool HandleMacroExpandedIdentifier(Token &Tok, MacroInfo *MI); - /// \brief Handle a module import directive. - void HandleModuleImport(Token &Import); - /// \brief Cache macro expanded tokens for TokenLexers. // /// Works like a stack; a TokenLexer adds the macro expanded tokens that is diff --git a/clang/lib/Lex/PPCaching.cpp b/clang/lib/Lex/PPCaching.cpp index 33106591c3ba..986341b98668 100644 --- a/clang/lib/Lex/PPCaching.cpp +++ b/clang/lib/Lex/PPCaching.cpp @@ -74,6 +74,8 @@ void Preprocessor::EnterCachingLexMode() { return; PushIncludeMacroStack(); + if (CurLexerKind != CLK_LexAfterModuleImport) + CurLexerKind = CLK_CachingLexer; } diff --git a/clang/lib/Lex/PPLexerChange.cpp b/clang/lib/Lex/PPLexerChange.cpp index 758bfc4bb0fd..d52220798e06 100644 --- a/clang/lib/Lex/PPLexerChange.cpp +++ b/clang/lib/Lex/PPLexerChange.cpp @@ -113,7 +113,9 @@ void Preprocessor::EnterSourceFileWithLexer(Lexer *TheLexer, CurLexer.reset(TheLexer); CurPPLexer = TheLexer; CurDirLookup = CurDir; - + if (CurLexerKind != CLK_LexAfterModuleImport) + CurLexerKind = CLK_Lexer; + // Notify the client, if desired, that we are in a new source file. if (Callbacks && !CurLexer->Is_PragmaLexer) { SrcMgr::CharacteristicKind FileType = @@ -135,7 +137,9 @@ void Preprocessor::EnterSourceFileWithPTH(PTHLexer *PL, CurDirLookup = CurDir; CurPTHLexer.reset(PL); CurPPLexer = CurPTHLexer.get(); - + if (CurLexerKind != CLK_LexAfterModuleImport) + CurLexerKind = CLK_PTHLexer; + // Notify the client, if desired, that we are in a new source file. if (Callbacks) { FileID FID = CurPPLexer->getFileID(); @@ -159,6 +163,8 @@ void Preprocessor::EnterMacro(Token &Tok, SourceLocation ILEnd, CurTokenLexer.reset(TokenLexerCache[--NumCachedTokenLexers]); CurTokenLexer->Init(Tok, ILEnd, Args); } + if (CurLexerKind != CLK_LexAfterModuleImport) + CurLexerKind = CLK_TokenLexer; } /// EnterTokenStream - Add a "macro" context to the top of the include stack, @@ -188,6 +194,8 @@ void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks, CurTokenLexer.reset(TokenLexerCache[--NumCachedTokenLexers]); CurTokenLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens); } + if (CurLexerKind != CLK_LexAfterModuleImport) + CurLexerKind = CLK_TokenLexer; } /// HandleEndOfFile - This callback is invoked when the lexer hits the end of diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp index d33d9c513da9..db3479e8f3eb 100644 --- a/clang/lib/Lex/Preprocessor.cpp +++ b/clang/lib/Lex/Preprocessor.cpp @@ -144,8 +144,6 @@ void Preprocessor::Initialize(const TargetInfo &Target) { // We haven't read anything from the external source. ReadMacrosFromExternalSource = false; - LexDepth = 0; - // "Poison" __VA_ARGS__, which can only appear in the expansion of a macro. // This gets unpoisoned where it is allowed. (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned(); @@ -524,27 +522,44 @@ void Preprocessor::HandleIdentifier(Token &Identifier) { // like "#define TY typeof", "TY(1) x". if (II.isExtensionToken() && !DisableMacroExpansion) Diag(Identifier, diag::ext_token_used); + + // If this is the '__import_module__' keyword, note that the next token + // indicates a module name. + if (II.getTokenID() == tok::kw___import_module__ && + !InMacroArgs && !DisableMacroExpansion) { + ModuleImportLoc = Identifier.getLocation(); + CurLexerKind = CLK_LexAfterModuleImport; + } } -void Preprocessor::HandleModuleImport(Token &Import) { +/// \brief Lex a token following the __import_module__ keyword. +void Preprocessor::LexAfterModuleImport(Token &Result) { + // Figure out what kind of lexer we actually have. + if (CurLexer) + CurLexerKind = CLK_Lexer; + else if (CurPTHLexer) + CurLexerKind = CLK_PTHLexer; + else if (CurTokenLexer) + CurLexerKind = CLK_TokenLexer; + else + CurLexerKind = CLK_CachingLexer; + + // Lex the next token. + Lex(Result); + // The token sequence // // __import_module__ identifier // - // indicates a module import directive. We load the module and then - // leave the token sequence for the parser. - Token ModuleNameTok = LookAhead(0); - if (ModuleNameTok.getKind() != tok::identifier) + // indicates a module import directive. We already saw the __import_module__ + // keyword, so now we're looking for the identifier. + if (Result.getKind() != tok::identifier) return; - (void)TheModuleLoader.loadModule(Import.getLocation(), - *ModuleNameTok.getIdentifierInfo(), - ModuleNameTok.getLocation()); - - // FIXME: Transmogrify __import_module__ into some kind of AST-only - // __import_module__ that is not recognized by the preprocessor but is - // recognized by the parser. It would also be useful to stash the ModuleKey - // somewhere, so we don't try to load the module twice. + // Load the module. + (void)TheModuleLoader.loadModule(ModuleImportLoc, + *Result.getIdentifierInfo(), + Result.getLocation()); } void Preprocessor::AddCommentHandler(CommentHandler *Handler) {