Implement "lparen scanning" for lexer buffers, by making "skipping lexing"

completely reversible.  This implements tests 3/4 of
test/Preprocessor/macro_fn_lparen_scan.c

llvm-svn: 38699

clang/Lex/Lexer.cpp

@@ -896,16 +896,21 @@ void Lexer::LexEndOfFile(LexerToken &Result, const char *CurPtr) {
     return;
   }        
 
-  // If we are in a #if directive, emit an error.
-  while (!ConditionalStack.empty()) {
-    PP.Diag(ConditionalStack.back().IfLoc,
-            diag::err_pp_unterminated_conditional);
-    ConditionalStack.pop_back();
-  }  
+  // If we aren't skipping, issue diagnostics.  If we are skipping, let the
+  // skipping code do this: there are multiple possible reasons for skipping,
+  // and not all want these diagnostics.
+  if (!PP.isSkipping()) {
+    // If we are in a #if directive, emit an error.
+    while (!ConditionalStack.empty()) {
+      PP.Diag(ConditionalStack.back().IfLoc,
+              diag::err_pp_unterminated_conditional);
+      ConditionalStack.pop_back();
+    }  
   
-  // If the file was empty or didn't end in a newline, issue a pedwarn.
-  if (CurPtr[-1] != '\n' && CurPtr[-1] != '\r')
-    Diag(BufferEnd, diag::ext_no_newline_eof);
+    // If the file was empty or didn't end in a newline, issue a pedwarn.
+    if (CurPtr[-1] != '\n' && CurPtr[-1] != '\r')
+      Diag(BufferEnd, diag::ext_no_newline_eof);
+  }
   
   BufferPtr = CurPtr;
   PP.HandleEndOfFile(Result);

clang/Lex/MacroExpander.cpp

@@ -79,18 +79,13 @@ void MacroExpander::Lex(LexerToken &Tok) {
   // Otherwise, return a normal token.
 }
 
-/// NextTokenIsKnownNotLParen - If the next token lexed will pop this macro
-/// off the expansion stack, return false and set RanOffEnd to true.
-/// Otherwise, return true if we know for sure that the next token returned
-/// will not be a '(' token.  Return false if it is a '(' token or if we are
-/// not sure.  This is used when determining whether to expand a function-like
-/// macro.
-bool MacroExpander::NextTokenIsKnownNotLParen(bool &RanOffEnd) const {
+/// isNextTokenLParen - If the next token lexed will pop this macro off the
+/// expansion stack, return 2.  If the next unexpanded token is a '(', return
+/// 1, otherwise return 0.
+unsigned MacroExpander::isNextTokenLParen() const {
   // Out of tokens?
-  if (CurToken == Macro.getNumTokens()) {
-    RanOffEnd = true;
-    return false;
-  }
+  if (CurToken == Macro.getNumTokens())
+    return 2;
 
-  return Macro.getReplacementToken(CurToken).getKind() != tok::l_paren;
+  return Macro.getReplacementToken(CurToken).getKind() == tok::l_paren;
 }

clang/Lex/Preprocessor.cpp

@@ -505,52 +505,72 @@ static bool isTrivialSingleTokenExpansion(const MacroInfo *MI,
     if (*I == II)
       return false;   // Identifier is a macro argument.
   return true;
-}  
+}
+
+/// isNextPPTokenLParen - Return 1 if the next unexpanded token lexed from
+/// the specified lexer will return a tok::l_paren token, 0 if it is something
+/// else and 2 if there are no more tokens in the buffer controlled by the
+/// lexer.
+unsigned Preprocessor::isNextPPTokenLParen(Lexer *L) {
+  assert(!isSkipping() && "How can we expand a macro from a skipping buffer?");
+  
+  // Set the lexer to 'skipping' mode.  This will ensure that we can lex a token
+  // without emitting diagnostics, disables macro expansion, and will cause EOF
+  // to return an EOF token instead of popping the include stack.
+  SkippingContents = true;
+  
+  // Save state that can be changed while lexing so that we can restore it.
+  const char *BufferPtr = L->BufferPtr;
+  
+  LexerToken Tok;
+  Tok.StartToken();
+  L->LexTokenInternal(Tok);
+
+  // Restore state that may have changed.
+  L->BufferPtr = BufferPtr;
+  
+  // Restore the lexer back to non-skipping mode.
+  SkippingContents = false;
+  
+  if (Tok.getKind() == tok::eof)
+    return 2;
+  return Tok.getKind() == tok::l_paren;
+}
+
 
 /// isNextPPTokenLParen - Determine whether the next preprocessor token to be
 /// lexed is a '('.  If so, consume the token and return true, if not, this
 /// method should have no observable side-effect on the lexed tokens.
 bool Preprocessor::isNextPPTokenLParen() {
-  bool RanOffEnd = false;
   // Do some quick tests for rejection cases.
-  if (CurLexer) {
-#if 0
-    if (!CurLexer->NextTokenIsKnownNotLParen(RanOffEnd))
-      return false;
-#endif
-  } else {
-    assert(CurMacroExpander && "No token source?");
-    if (CurMacroExpander->NextTokenIsKnownNotLParen(RanOffEnd))
-      return false;
+  unsigned Val;
+  if (CurLexer)
+    Val = isNextPPTokenLParen(CurLexer);
+  else
+    Val = CurMacroExpander->isNextTokenLParen();
+  
+  if (Val == 2) {
+    // If we ran off the end of the lexer or macro expander, walk the include
+    // stack, looking for whatever will return the next token.
+    for (unsigned i = IncludeMacroStack.size(); Val == 2 && i != 0; --i) {
+      IncludeStackInfo &Entry = IncludeMacroStack[i-1];
+      if (Entry.TheLexer)
+        Val = isNextPPTokenLParen(Entry.TheLexer);
+      else
+        Val = Entry.TheMacroExpander->isNextTokenLParen();
+    }
   }
 
-  // If we ran off the end of the lexer or macro expander, walk the include
-  // stack, looking for whatever will return the next token.
-  for (unsigned i = IncludeMacroStack.size(); RanOffEnd && i != 0; --i) {
-    IncludeStackInfo &Entry = IncludeMacroStack[i-1];
-    RanOffEnd = false;
-    if (Entry.TheLexer) {
-#if 0
-      if (!Entry.TheLexer->NextTokenIsKnownNotLParen(RanOffEnd))
-        return false;
-#endif
-    } else if (Entry.TheMacroExpander->NextTokenIsKnownNotLParen(RanOffEnd))
-      return false;
-  }
-
-  // Okay, if we get here we either know that the next token definitely IS a '('
-  // token, or we don't know what it is.  In either case we will speculatively
-  // read the next token.  If it turns out that it isn't a '(', then we create a
-  // new macro context with just that token on it so that the token gets
-  // reprocessed.
+  // Okay, if we know that the token is a '(', lex it and return.  Otherwise we
+  // have found something that isn't a '(' or we found the end of the
+  // translation unit.  In either case, return false.
+  if (Val != 1)
+    return false;
   
   LexerToken Tok;
   LexUnexpandedToken(Tok);
-  if (Tok.getKind() == tok::l_paren)
-    return true;
-  
-  // FIXME: push a fake macro context, push Tok onto it.
-  assert(0 && "FIXME: implement speculation failure code!");
+  assert(Tok.getKind() == tok::l_paren && "Error computing l-paren-ness?");
+  return true;
 }
 
 /// HandleMacroExpandedIdentifier - If an identifier token is read that is to be
@@ -984,8 +1004,9 @@ void Preprocessor::HandleEndOfFile(LexerToken &Result, bool isEndOfMacro) {
   
   // If we are in a #if 0 block skipping tokens, and we see the end of the file,
   // this is an error condition.  Just return the EOF token up to
-  // SkipExcludedConditionalBlock.  The Lexer will have already have issued
-  // errors for the unterminated #if's on the conditional stack.
+  // SkipExcludedConditionalBlock.  The code that enabled skipping will issue
+  // errors for the unterminated #if's on the conditional stack if it is
+  // interested.
   if (isSkipping()) {
     Result.StartToken();
     CurLexer->BufferPtr = CurLexer->BufferEnd;
@@ -1165,10 +1186,19 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
   while (1) {
     CurLexer->Lex(Tok);
     
-    // If this is the end of the buffer, we have an error.  The lexer will have
-    // already handled this error condition, so just return and let the caller
-    // lex after this #include.
-    if (Tok.getKind() == tok::eof) break;
+    // If this is the end of the buffer, we have an error.
+    if (Tok.getKind() == tok::eof) {
+      // Emit errors for each unterminated conditional on the stack, including
+      // the current one.
+      while (!CurLexer->ConditionalStack.empty()) {
+        Diag(CurLexer->ConditionalStack.back().IfLoc,
+             diag::err_pp_unterminated_conditional);
+        CurLexer->ConditionalStack.pop_back();
+      }  
+      
+      // Just return and let the caller lex after this #include.
+      break;
+    }
     
     // If this token is not a preprocessor directive, just skip it.
     if (Tok.getKind() != tok::hash || !Tok.isAtStartOfLine())

clang/include/clang/Lex/Lexer.h

@@ -62,12 +62,14 @@ class Lexer {
   bool IsMainFile;               // True if top-level file.
   
   // Context-specific lexing flags.
-  bool IsAtStartOfLine;          // True if sitting at start of line.
   bool ParsingPreprocessorDirective; // True if parsing #XXX
   bool ParsingFilename;          // True after #include: turn <xx> into string.
   
-  // Context that changes as the file is lexed.
-  
+  // Context that changes as the file is lexed.  NOTE: any state that mutates as
+  // the file is lexed should be added to Preprocessor::isNextPPTokenLParen.
+
+  bool IsAtStartOfLine;          // True if sitting at start of line.
+
   /// MIOpt - This is a state machine that detects the #ifndef-wrapping a file 
   /// idiom for the multiple-include optimization.
   MultipleIncludeOpt MIOpt;

clang/include/clang/Lex/MacroExpander.h

@@ -82,13 +82,10 @@ public:
     delete FormalArgs;
   }
   
-  /// NextTokenIsKnownNotLParen - If the next token lexed will pop this macro
-  /// off the expansion stack, return false and set RanOffEnd to true.
-  /// Otherwise, return true if we know for sure that the next token returned
-  /// will not be a '(' token.  Return false if it is a '(' token or if we are
-  /// not sure.  This is used when determining whether to expand a function-like
-  /// macro.
-  bool NextTokenIsKnownNotLParen(bool &RanOffEnd) const;
+  /// isNextTokenLParen - If the next token lexed will pop this macro off the
+  /// expansion stack, return 2.  If the next unexpanded token is a '(', return
+  /// 1, otherwise return 0.
+  unsigned isNextTokenLParen() const;
   
   MacroInfo &getMacro() const { return Macro; }
 

clang/include/clang/Lex/Preprocessor.h

@@ -467,6 +467,12 @@ private:
   /// the macro should not be expanded return true, otherwise return false.
   bool HandleMacroExpandedIdentifier(LexerToken &Tok, MacroInfo *MI);
   
+  /// isNextPPTokenLParen - Return 1 if the next unexpanded token lexed from
+  /// the specified lexer will return a tok::l_paren token, 0 if it is something
+  /// else and 2 if there are no more tokens in the buffer controlled by the
+  /// lexer.
+  unsigned isNextPPTokenLParen(Lexer *L);
+  
   /// isNextPPTokenLParen - Determine whether the next preprocessor token to be
   /// lexed is a '('.  If so, consume the token and return true, if not, this
   /// method should have no observable side-effect on the lexed tokens.