Parse: Don't parse after the eof has been consumed

ParseCXXNonStaticMemberInitializer stashes away all the tokens for the
initializer and an additional EOF token to denote where the initializer
ends.  However, it is possible for ParseLexedMemberInitializer to get
its hands on the "real" EOF token; since the two tokens are
indistinguishable, we end up consuming the EOF and descend into madness.

Instead, make it possible to tell which EOF token we are looking at.

This fixes PR21872.

llvm-svn: 224505

clang/include/clang/Lex/Token.h

@@ -23,6 +23,7 @@
 
 namespace clang {
 
+class Decl;
 class IdentifierInfo;
 
 /// Token - This structure provides full information about a lexed token.
@@ -58,6 +59,8 @@ class Token {
   ///    may be dirty (have trigraphs / escaped newlines).
   ///  Annotations (resolved type names, C++ scopes, etc): isAnnotation().
   ///    This is a pointer to sema-specific data for the annotation token.
+  ///  Eof:
+  //     This is a pointer to a Decl.
   ///  Other:
   ///    This is null.
   void *PtrData;
@@ -164,12 +167,23 @@ public:
     assert(!isAnnotation() &&
            "getIdentifierInfo() on an annotation token!");
     if (isLiteral()) return nullptr;
+    if (is(tok::eof)) return nullptr;
     return (IdentifierInfo*) PtrData;
   }
   void setIdentifierInfo(IdentifierInfo *II) {
     PtrData = (void*) II;
   }
 
+  const Decl *getDecl() const {
+    assert(is(tok::eof));
+    return reinterpret_cast<const Decl *>(PtrData);
+  }
+  void setDecl(const Decl *D) {
+    assert(is(tok::eof));
+    assert(!PtrData);
+    PtrData = const_cast<Decl *>(D);
+  }
+
   /// getRawIdentifier - For a raw identifier token (i.e., an identifier
   /// lexed in raw mode), returns a reference to the text substring in the
   /// buffer if known.

clang/lib/Parse/ParseCXXInlineMethods.cpp

@@ -218,6 +218,7 @@ void Parser::ParseCXXNonStaticMemberInitializer(Decl *VarD) {
   Eof.startToken();
   Eof.setKind(tok::eof);
   Eof.setLocation(Tok.getLocation());
+  Eof.setDecl(VarD);
   Toks.push_back(Eof);
 }
 
@@ -622,7 +623,9 @@ void Parser::ParseLexedMemberInitializer(LateParsedMemberInitializer &MI) {
     while (Tok.isNot(tok::eof))
       ConsumeAnyToken();
   }
-  ConsumeAnyToken();
+  // Make sure this is *our* artificial EOF token.
+  if (Tok.getDecl() == MI.Field)
+    ConsumeAnyToken();
 }
 
 /// ConsumeAndStoreUntil - Consume and store the token at the passed token

clang/test/Parser/PR21872.cpp

@@ -0,0 +1,4 @@
+// RUN: not %clang_cc1 -fsyntax-only %s
+template <typename T> struct S {
+    int k = ((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((
+int f;