Support code-completion for C++ inline methods and ObjC buffering methods.

Previously we would cut off the source file buffer at the code-completion
point; this impeded code-completion inside C++ inline methods and,
recently, with buffering ObjC methods.

Have the code-completion inserted into the source buffer so that it can
be buffered along with a method body. When we actually hit the code-completion
point the cut-off lexing or parsing.

Fixes rdar://10056932&8319466

llvm-svn: 139086

clang/include/clang/Lex/Lexer.h

@@ -500,6 +500,9 @@ private:
   
   bool IsStartOfConflictMarker(const char *CurPtr);
   bool HandleEndOfConflictMarker(const char *CurPtr);
+
+  bool isCodeCompletionPoint(const char *CurPtr) const;
+  void cutOffLexing() { BufferPtr = BufferEnd; }
 };
 
 

clang/include/clang/Lex/Preprocessor.h

@@ -152,6 +152,21 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   /// \brief The file that we're performing code-completion for, if any.
   const FileEntry *CodeCompletionFile;
 
+  /// \brief The offset in file for the code-completion point.
+  unsigned CodeCompletionOffset;
+
+  /// \brief The location for the code-completion point. This gets instantiated
+  /// when the CodeCompletionFile gets #include'ed for preprocessing.
+  SourceLocation CodeCompletionLoc;
+
+  /// \brief The start location for the file of the code-completion point.
+  /// This gets instantiated when the CodeCompletionFile gets #include'ed
+  /// for preprocessing.
+  SourceLocation CodeCompletionFileLoc;
+
+  /// \brief True if we hit the code-completion point.
+  bool CodeCompletionReached;
+
   /// \brief The number of bytes that we will initially skip when entering the
   /// main file, which is used when loading a precompiled preamble, along
   /// with a flag that indicates whether skipping this number of bytes will
@@ -665,13 +680,33 @@ public:
   bool SetCodeCompletionPoint(const FileEntry *File,
                               unsigned Line, unsigned Column);
 
-  /// \brief Determine if this source location refers into the file
-  /// for which we are performing code completion.
-  bool isCodeCompletionFile(SourceLocation FileLoc) const;
-
   /// \brief Determine if we are performing code completion.
   bool isCodeCompletionEnabled() const { return CodeCompletionFile != 0; }
 
+  /// \brief Returns the location of the code-completion point.
+  /// Returns an invalid location if code-completion is not enabled or the file
+  /// containing the code-completion point has not been lexed yet.
+  SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
+
+  /// \brief Returns the start location of the file of code-completion point.
+  /// Returns an invalid location if code-completion is not enabled or the file
+  /// containing the code-completion point has not been lexed yet.
+  SourceLocation getCodeCompletionFileLoc() const {
+    return CodeCompletionFileLoc;
+  }
+
+  /// \brief Returns true if code-completion is enabled and we have hit the
+  /// code-completion point.
+  bool isCodeCompletionReached() const { return CodeCompletionReached; }
+
+  /// \brief Note that we hit the code-completion point.
+  void setCodeCompletionReached() {
+    assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
+    CodeCompletionReached = true;
+    // Silence any diagnostics that occur after we hit the code-completion.
+    getDiagnostics().setSuppressAllDiagnostics(true);
+  }
+
   /// \brief Instruct the preprocessor to skip part of the main
   /// the main source file.
   ///

clang/include/clang/Parse/Parser.h

@@ -286,10 +286,9 @@ private:
     assert(!isTokenStringLiteral() && !isTokenParen() && !isTokenBracket() &&
            !isTokenBrace() &&
            "Should consume special tokens with Consume*Token");
-    if (Tok.is(tok::code_completion)) {
+
-      CodeCompletionRecovery();
+    if (Tok.is(tok::code_completion))
-      return ConsumeCodeCompletionToken();
+      return handleUnexpectedCodeCompletionToken();
-    }
 
     PrevTokLocation = Tok.getLocation();
     PP.Lex(Tok);
@@ -376,10 +375,20 @@ private:
     return PrevTokLocation;    
   }
   
-  ///\ brief When we are consuming a code-completion token within having 
+  ///\ brief When we are consuming a code-completion token without having
   /// matched specific position in the grammar, provide code-completion results
   /// based on context.
-  void CodeCompletionRecovery();
+  ///
+  /// \returns the source location of the code-completion token.
+  SourceLocation handleUnexpectedCodeCompletionToken();
+
+  /// \brief Abruptly cut off parsing; mainly used when we have reached the
+  /// code-completion point.
+  void cutOffParsing() {
+    PP.setCodeCompletionReached();
+    // Cut off parsing by acting as if we reached the end-of-file.
+    Tok.setKind(tok::eof);
+  }
 
   /// \brief Handle the annotation token produced for #pragma unused(...)
   void HandlePragmaUnused();

clang/lib/Lex/Lexer.cpp

@@ -1384,16 +1384,21 @@ void Lexer::LexStringLiteral(Token &Result, const char *CurPtr,
     
     if (C == '\n' || C == '\r' ||             // Newline.
         (C == 0 && CurPtr-1 == BufferEnd)) {  // End of file.
-      if (C == 0 && PP && PP->isCodeCompletionFile(FileLoc))
+      if (!isLexingRawMode() && !Features.AsmPreprocessor)
-        PP->CodeCompleteNaturalLanguage();
-      else if (!isLexingRawMode() && !Features.AsmPreprocessor)
         Diag(BufferPtr, diag::warn_unterminated_string);
       FormTokenWithChars(Result, CurPtr-1, tok::unknown);
       return;
     }
     
-    if (C == 0)
+    if (C == 0) {
+      if (isCodeCompletionPoint(CurPtr-1)) {
+        PP->CodeCompleteNaturalLanguage();
+        FormTokenWithChars(Result, CurPtr-1, tok::unknown);
+        return cutOffLexing();
+      }
+
       NulCharacter = CurPtr-1;
+    }
     C = getAndAdvanceChar(CurPtr, Result);
   }
 
@@ -1491,7 +1496,8 @@ void Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) {
       // Skip the escaped character.
       C = getAndAdvanceChar(CurPtr, Result);
     } else if (C == '\n' || C == '\r' ||             // Newline.
-               (C == 0 && CurPtr-1 == BufferEnd)) {  // End of file.
+               (C == 0 && (CurPtr-1 == BufferEnd ||  // End of file.
+                           isCodeCompletionPoint(CurPtr-1)))) {
       // If the filename is unterminated, then it must just be a lone <
       // character.  Return this as such.
       FormTokenWithChars(Result, AfterLessPos, tok::less);
@@ -1535,13 +1541,17 @@ void Lexer::LexCharConstant(Token &Result, const char *CurPtr,
       C = getAndAdvanceChar(CurPtr, Result);
     } else if (C == '\n' || C == '\r' ||             // Newline.
                (C == 0 && CurPtr-1 == BufferEnd)) {  // End of file.
-      if (C == 0 && PP && PP->isCodeCompletionFile(FileLoc))
+      if (!isLexingRawMode() && !Features.AsmPreprocessor)
-        PP->CodeCompleteNaturalLanguage();
-      else if (!isLexingRawMode() && !Features.AsmPreprocessor)
         Diag(BufferPtr, diag::warn_unterminated_char);
       FormTokenWithChars(Result, CurPtr-1, tok::unknown);
       return;
     } else if (C == 0) {
+      if (isCodeCompletionPoint(CurPtr-1)) {
+        PP->CodeCompleteNaturalLanguage();
+        FormTokenWithChars(Result, CurPtr-1, tok::unknown);
+        return cutOffLexing();
+      }
+
       NulCharacter = CurPtr-1;
     }
     C = getAndAdvanceChar(CurPtr, Result);
@@ -1682,12 +1692,16 @@ bool Lexer::SkipBCPLComment(Token &Result, const char *CurPtr) {
     }
 
     if (CurPtr == BufferEnd+1) { 
-      if (PP && PP->isCodeCompletionFile(FileLoc))
-        PP->CodeCompleteNaturalLanguage();
-
       --CurPtr; 
       break; 
     }
+
+    if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) {
+      PP->CodeCompleteNaturalLanguage();
+      cutOffLexing();
+      return false;
+    }
+
   } while (C != '\n' && C != '\r');
 
   // Found but did not consume the newline.  Notify comment handlers about the
@@ -1842,8 +1856,7 @@ bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) {
   unsigned char C = getCharAndSize(CurPtr, CharSize);
   CurPtr += CharSize;
   if (C == 0 && CurPtr == BufferEnd+1) {
-    if (!isLexingRawMode() &&
+    if (!isLexingRawMode())
-        !PP->isCodeCompletionFile(FileLoc))
       Diag(BufferPtr, diag::err_unterminated_block_comment);
     --CurPtr;
 
@@ -1866,7 +1879,10 @@ bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) {
   while (1) {
     // Skip over all non-interesting characters until we find end of buffer or a
     // (probably ending) '/' character.
-    if (CurPtr + 24 < BufferEnd) {
+    if (CurPtr + 24 < BufferEnd &&
+        // If there is a code-completion point avoid the fast scan because it
+        // doesn't check for '\0'.
+        !(PP && PP->getCodeCompletionFileLoc() == FileLoc)) {
       // While not aligned to a 16-byte boundary.
       while (C != '/' && ((intptr_t)CurPtr & 0x0F) != 0)
         C = *CurPtr++;
@@ -1926,9 +1942,7 @@ bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) {
           Diag(CurPtr-1, diag::warn_nested_block_comment);
       }
     } else if (C == 0 && CurPtr == BufferEnd+1) {
-      if (PP && PP->isCodeCompletionFile(FileLoc))
+      if (!isLexingRawMode())
-        PP->CodeCompleteNaturalLanguage();
-      else if (!isLexingRawMode())
         Diag(BufferPtr, diag::err_unterminated_block_comment);
       // Note: the user probably forgot a */.  We could continue immediately
       // after the /*, but this would involve lexing a lot of what really is the
@@ -1944,7 +1958,12 @@ bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) {
 
       BufferPtr = CurPtr;
       return false;
+    } else if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) {
+      PP->CodeCompleteNaturalLanguage();
+      cutOffLexing();
+      return false;
     }
+
     C = *CurPtr++;
   }
 
@@ -2001,6 +2020,12 @@ std::string Lexer::ReadToEndOfLine() {
     case 0:  // Null.
       // Found end of file?
       if (CurPtr-1 != BufferEnd) {
+        if (isCodeCompletionPoint(CurPtr-1)) {
+          PP->CodeCompleteNaturalLanguage();
+          cutOffLexing();
+          return Result;
+        }
+
         // Nope, normal character, continue.
         Result += Char;
         break;
@@ -2015,8 +2040,8 @@ std::string Lexer::ReadToEndOfLine() {
       // Next, lex the character, which should handle the EOD transition.
       Lex(Tmp);
       if (Tmp.is(tok::code_completion)) {
-        if (PP && PP->getCodeCompletionHandler())
+        if (PP)
-          PP->getCodeCompletionHandler()->CodeCompleteNaturalLanguage();
+          PP->CodeCompleteNaturalLanguage();
         Lex(Tmp);
       }
       assert(Tmp.is(tok::eod) && "Unexpected token!");
@@ -2032,22 +2057,6 @@ std::string Lexer::ReadToEndOfLine() {
 /// This returns true if Result contains a token, false if PP.Lex should be
 /// called again.
 bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) {
-  // Check if we are performing code completion.
-  if (PP && PP->isCodeCompletionFile(FileLoc)) {
-    // We're at the end of the file, but we've been asked to consider the
-    // end of the file to be a code-completion token. Return the
-    // code-completion token.
-    Result.startToken();
-    FormTokenWithChars(Result, CurPtr, tok::code_completion);
-    
-    // Only do the eof -> code_completion translation once.
-    PP->SetCodeCompletionPoint(0, 0, 0);
-    
-    // Silence any diagnostics that occur once we hit the code-completion point.
-    PP->getDiagnostics().setSuppressAllDiagnostics(true);
-    return true;
-  }
-
   // If we hit the end of the file while parsing a preprocessor directive,
   // end the preprocessor directive first.  The next token returned will
   // then be the end of file.
@@ -2075,7 +2084,7 @@ bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) {
 
   // If we are in a #if directive, emit an error.
   while (!ConditionalStack.empty()) {
-    if (!PP->isCodeCompletionFile(FileLoc))
+    if (PP->getCodeCompletionFileLoc() != FileLoc)
       PP->Diag(ConditionalStack.back().IfLoc,
                diag::err_pp_unterminated_conditional);
     ConditionalStack.pop_back();
@@ -2225,6 +2234,15 @@ bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) {
   return false;
 }
 
+bool Lexer::isCodeCompletionPoint(const char *CurPtr) const {
+  if (PP && PP->isCodeCompletionEnabled()) {
+    SourceLocation Loc = FileLoc.getFileLocWithOffset(CurPtr-BufferStart);
+    return Loc == PP->getCodeCompletionLoc();
+  }
+
+  return false;
+}
+
 
 /// LexTokenInternal - This implements a simple C family lexer.  It is an
 /// extremely performance critical piece of code.  This assumes that the buffer
@@ -2277,6 +2295,14 @@ LexNextToken:
       return PPCache->Lex(Result);
     }
 
+    // Check if we are performing code completion.
+    if (isCodeCompletionPoint(CurPtr-1)) {
+      // Return the code-completion token.
+      Result.startToken();
+      FormTokenWithChars(Result, CurPtr, tok::code_completion);
+      return;
+    }
+
     if (!isLexingRawMode())
       Diag(CurPtr-1, diag::null_in_file);
     Result.setFlag(Token::LeadingSpace);

clang/lib/Lex/PPDirectives.cpp

@@ -102,8 +102,8 @@ void Preprocessor::ReadMacroName(Token &MacroNameTok, char isDefineUndef) {
   if (MacroNameTok.is(tok::code_completion)) {
     if (CodeComplete)
       CodeComplete->CodeCompleteMacroName(isDefineUndef == 1);
+    setCodeCompletionReached();
     LexUnexpandedToken(MacroNameTok);
-    return;
   }
   
   // Missing macro name?
@@ -214,6 +214,7 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
     if (Tok.is(tok::code_completion)) {
       if (CodeComplete)
         CodeComplete->CodeCompleteInConditionalExclusion();
+      setCodeCompletionReached();
       continue;
     }
     
@@ -222,7 +223,7 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
       // Emit errors for each unterminated conditional on the stack, including
       // the current one.
       while (!CurPPLexer->ConditionalStack.empty()) {
-        if (!isCodeCompletionFile(Tok.getLocation()))
+        if (CurLexer->getFileLoc() != CodeCompletionFileLoc)
           Diag(CurPPLexer->ConditionalStack.back().IfLoc,
                diag::err_pp_unterminated_conditional);
         CurPPLexer->ConditionalStack.pop_back();
@@ -581,6 +582,7 @@ TryAgain:
     if (CodeComplete)
       CodeComplete->CodeCompleteDirective(
                                     CurPPLexer->getConditionalStackDepth() > 0);
+    setCodeCompletionReached();
     return;
   case tok::numeric_constant:  // # 7  GNU line marker directive.
     if (getLangOptions().AsmPreprocessor)
@@ -1104,6 +1106,7 @@ bool Preprocessor::ConcatenateIncludeName(
     
     // FIXME: Provide code completion for #includes.
     if (CurTok.is(tok::code_completion)) {
+      setCodeCompletionReached();
       Lex(CurTok);
       continue;
     }

clang/lib/Lex/PPExpressions.cpp

@@ -96,6 +96,7 @@ static bool EvaluateDefined(PPValue &Result, Token &PeekTok, DefinedTracker &DT,
   if (PeekTok.is(tok::code_completion)) {
     if (PP.getCodeCompletionHandler())
       PP.getCodeCompletionHandler()->CodeCompleteMacroName(false);
+    PP.setCodeCompletionReached();
     PP.LexUnexpandedNonComment(PeekTok);
   }
   
@@ -156,6 +157,7 @@ static bool EvaluateValue(PPValue &Result, Token &PeekTok, DefinedTracker &DT,
   if (PeekTok.is(tok::code_completion)) {
     if (PP.getCodeCompletionHandler())
       PP.getCodeCompletionHandler()->CodeCompletePreprocessorExpression();
+    PP.setCodeCompletionReached();
     PP.LexNonComment(PeekTok);
   }
       

clang/lib/Lex/PPLexerChange.cpp

@@ -90,6 +90,13 @@ void Preprocessor::EnterSourceFile(FileID FID, const DirectoryLookup *CurDir,
     return;
   }
 
+  if (isCodeCompletionEnabled() &&
+      SourceMgr.getFileEntryForID(FID) == CodeCompletionFile) {
+    CodeCompletionFileLoc = SourceMgr.getLocForStartOfFile(FID);
+    CodeCompletionLoc =
+        CodeCompletionFileLoc.getFileLocWithOffset(CodeCompletionOffset);
+  }
+
   EnterSourceFileWithLexer(new Lexer(FID, InputFile, *this), CurDir);
   return;
 }
@@ -204,6 +211,25 @@ bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) {
   // If this is a #include'd file, pop it off the include stack and continue
   // lexing the #includer file.
   if (!IncludeMacroStack.empty()) {
+
+    // If we lexed the code-completion file, act as if we reached EOF.
+    if (isCodeCompletionEnabled() && CurPPLexer &&
+        SourceMgr.getLocForStartOfFile(CurPPLexer->getFileID()) ==
+            CodeCompletionFileLoc) {
+      if (CurLexer) {
+        Result.startToken();
+        CurLexer->FormTokenWithChars(Result, CurLexer->BufferEnd, tok::eof);
+        CurLexer.reset();
+      } else {
+        assert(CurPTHLexer && "Got EOF but no current lexer set!");
+        CurPTHLexer->getEOF(Result);
+        CurPTHLexer.reset();
+      }
+
+      CurPPLexer = 0;
+      return true;
+    }
+
     if (!isEndOfMacro && CurPPLexer &&
         SourceMgr.getIncludeLoc(CurPPLexer->getFileID()).isValid()) {
       // Notify SourceManager to record the number of FileIDs that were created

clang/lib/Lex/PPMacroExpansion.cpp

@@ -353,29 +353,6 @@ MacroArgs *Preprocessor::ReadFunctionLikeMacroArgs(Token &MacroName,
       // an argument value in a macro could expand to ',' or '(' or ')'.
       LexUnexpandedToken(Tok);
 
-      if (Tok.is(tok::code_completion)) {
-        if (CodeComplete)
-          CodeComplete->CodeCompleteMacroArgument(MacroName.getIdentifierInfo(),
-                                                  MI, NumActuals);
-
-        // Add the code-completion token and finish the lexing normally so that
-        // normal code-completion occurs again with the expanded tokens.
-        ArgTokens.push_back(Tok);
-        // Add a marker EOF token to the end of the token list.
-        Token EOFTok;
-        EOFTok.startToken();
-        EOFTok.setKind(tok::eof);
-        EOFTok.setLocation(Tok.getLocation());
-        EOFTok.setLength(0);
-        ArgTokens.push_back(EOFTok);
-        ++NumActuals;
-        // "Fill out" the other arguments.
-        for (; NumActuals < MI->getNumArgs(); ++NumActuals)
-          ArgTokens.push_back(EOFTok);
-        return MacroArgs::create(MI, ArgTokens.data(), ArgTokens.size(),
-                                 /*isVarargsElided=*/false, *this);
-      }
-      
       if (Tok.is(tok::eof) || Tok.is(tok::eod)) { // "#if f(<eof>" & "#if f(\n"
         Diag(MacroName, diag::err_unterm_macro_invoc);
         // Do not lose the EOF/EOD.  Return it to the client.
@@ -410,7 +387,15 @@ MacroArgs *Preprocessor::ReadFunctionLikeMacroArgs(Token &MacroName,
         if (MacroInfo *MI = getMacroInfo(Tok.getIdentifierInfo()))
           if (!MI->isEnabled())
             Tok.setFlag(Token::DisableExpand);
+      } else if (Tok.is(tok::code_completion)) {
+        if (CodeComplete)
+          CodeComplete->CodeCompleteMacroArgument(MacroName.getIdentifierInfo(),
+                                                  MI, NumActuals);
+        // Don't mark that we reached the code-completion point because the
+        // parser is going to handle the token and there will be another
+        // code-completion callback.
       }
+
       ArgTokens.push_back(Tok);
     }
 

clang/lib/Lex/PTHLexer.cpp

@@ -147,7 +147,7 @@ bool PTHLexer::LexEndOfFile(Token &Result) {
 
   // If we are in a #if directive, emit an error.
   while (!ConditionalStack.empty()) {
-    if (!PP->isCodeCompletionFile(FileStartLoc))
+    if (PP->getCodeCompletionFileLoc() != FileStartLoc)
       PP->Diag(ConditionalStack.back().IfLoc,
                diag::err_pp_unterminated_conditional);
     ConditionalStack.pop_back();

clang/lib/Lex/Preprocessor.cpp

@@ -59,7 +59,8 @@ Preprocessor::Preprocessor(Diagnostic &diags, LangOptions &opts,
     SourceMgr(SM), HeaderInfo(Headers), TheModuleLoader(TheModuleLoader),
     ExternalSource(0), 
     Identifiers(opts, IILookup), CodeComplete(0),
-    CodeCompletionFile(0), SkipMainFilePreamble(0, true), CurPPLexer(0), 
+    CodeCompletionFile(0), CodeCompletionOffset(0), CodeCompletionReached(0),
+    SkipMainFilePreamble(0, true), CurPPLexer(0), 
     CurDirLookup(0), Callbacks(0), MacroArgCache(0), Record(0), MIChainHead(0),
     MICache(0) 
 {
@@ -73,7 +74,8 @@ Preprocessor::Preprocessor(Diagnostic &diags, LangOptions &opts,
 
 Preprocessor::~Preprocessor() {
   assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
-  assert(MacroExpandingLexersStack.empty() && MacroExpandedTokens.empty() &&
+  assert(((MacroExpandingLexersStack.empty() && MacroExpandedTokens.empty()) ||
+          isCodeCompletionReached()) &&
          "Preprocessor::HandleEndOfTokenLexer should have cleared those");
 
   while (!IncludeMacroStack.empty()) {
@@ -270,16 +272,14 @@ Preprocessor::macro_end(bool IncludeExternalMacros) const {
 }
 
 bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File,
-                                          unsigned TruncateAtLine,
+                                          unsigned CompleteLine,
-                                          unsigned TruncateAtColumn) {
+                                          unsigned CompleteColumn) {
+  assert(File);
+  assert(CompleteLine && CompleteColumn && "Starts from 1:1");
+  assert(!CodeCompletionFile && "Already set");
+
   using llvm::MemoryBuffer;
 
-  CodeCompletionFile = File;
-
-  // Okay to clear out the code-completion point by passing NULL.
-  if (!CodeCompletionFile)
-    return false;
-
   // Load the actual file's contents.
   bool Invalid = false;
   const MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File, &Invalid);
@@ -288,7 +288,7 @@ bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File,
 
   // Find the byte position of the truncation point.
   const char *Position = Buffer->getBufferStart();
-  for (unsigned Line = 1; Line < TruncateAtLine; ++Line) {
+  for (unsigned Line = 1; Line < CompleteLine; ++Line) {
     for (; *Position; ++Position) {
       if (*Position != '\r' && *Position != '\n')
         continue;
@@ -302,31 +302,30 @@ bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File,
     }
   }
 
-  Position += TruncateAtColumn - 1;
+  Position += CompleteColumn - 1;
 
-  // Truncate the buffer.
+  // Insert '\0' at the code-completion point.
   if (Position < Buffer->getBufferEnd()) {
-    StringRef Data(Buffer->getBufferStart(),
+    CodeCompletionFile = File;
-                         Position-Buffer->getBufferStart());
+    CodeCompletionOffset = Position - Buffer->getBufferStart();
-    MemoryBuffer *TruncatedBuffer
+
-      = MemoryBuffer::getMemBufferCopy(Data, Buffer->getBufferIdentifier());
+    MemoryBuffer *NewBuffer =
-    SourceMgr.overrideFileContents(File, TruncatedBuffer);
+        MemoryBuffer::getNewUninitMemBuffer(Buffer->getBufferSize() + 1,
+                                            Buffer->getBufferIdentifier());
+    char *NewBuf = (char*)NewBuffer->getBufferStart();
+    char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
+    *NewPos = '\0';
+    std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
+    SourceMgr.overrideFileContents(File, NewBuffer);
   }
 
   return false;
 }
 
-bool Preprocessor::isCodeCompletionFile(SourceLocation FileLoc) const {
-  return CodeCompletionFile && FileLoc.isFileID() &&
-    SourceMgr.getFileEntryForID(SourceMgr.getFileID(FileLoc))
-      == CodeCompletionFile;
-}
-
 void Preprocessor::CodeCompleteNaturalLanguage() {
-  SetCodeCompletionPoint(0, 0, 0);
-  getDiagnostics().setSuppressAllDiagnostics(true);
   if (CodeComplete)
     CodeComplete->CodeCompleteNaturalLanguage();
+  setCodeCompletionReached();
 }
 
 /// getSpelling - This method is used to get the spelling of a token into a

clang/lib/Parse/ParseCXXInlineMethods.cpp

@@ -551,6 +551,11 @@ bool Parser::ConsumeAndStoreUntil(tok::TokenKind T1, tok::TokenKind T2,
       ConsumeBrace();
       break;
 
+    case tok::code_completion:
+      Toks.push_back(Tok);
+      ConsumeCodeCompletionToken();
+      break;
+
     case tok::string_literal:
     case tok::wide_string_literal:
     case tok::utf8_string_literal:

clang/lib/Parse/ParseDecl.cpp

@@ -1084,9 +1084,8 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(Declarator &D,
 
       if (Tok.is(tok::code_completion)) {
         Actions.CodeCompleteInitializer(getCurScope(), ThisDecl);
-        ConsumeCodeCompletionToken();
+        cutOffParsing();
-        SkipUntil(tok::comma, true, true);
+        return 0;
-        return ThisDecl;
       }
       
       ExprResult Init(ParseInitializer());
@@ -1423,8 +1422,7 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
         Actions.CodeCompleteDeclSpec(getCurScope(), DS,
                                      AllowNonIdentifiers, 
                                      AllowNestedNameSpecifiers);
-        ConsumeCodeCompletionToken();
+        return cutOffParsing();
-        return;
       } 
       
       if (getCurScope()->getFnParent() || getCurScope()->getBlockParent())
@@ -1438,8 +1436,7 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
         CCC = Sema::PCC_ObjCImplementation;
       
       Actions.CodeCompleteOrdinaryName(getCurScope(), CCC);
-      ConsumeCodeCompletionToken();
+      return cutOffParsing();
-      return;
     }
 
     case tok::coloncolon: // ::foo::bar
@@ -2539,7 +2536,7 @@ void Parser::ParseEnumSpecifier(SourceLocation StartLoc, DeclSpec &DS,
   if (Tok.is(tok::code_completion)) {
     // Code completion for an enum name.
     Actions.CodeCompleteTag(getCurScope(), DeclSpec::TST_enum);
-    ConsumeCodeCompletionToken();
+    return cutOffParsing();
   }
 
   bool IsScopedEnum = false;
@@ -3243,8 +3240,7 @@ void Parser::ParseTypeQualifierListOpt(DeclSpec &DS,
     switch (Tok.getKind()) {
     case tok::code_completion:
       Actions.CodeCompleteTypeQualifiers(DS);
-      ConsumeCodeCompletionToken();
+      return cutOffParsing();
-      break;
         
     case tok::kw_const:
       isInvalid = DS.SetTypeQual(DeclSpec::TQ_const   , Loc, PrevSpec, DiagID,

clang/lib/Parse/ParseDeclCXX.cpp

@@ -56,7 +56,8 @@ Decl *Parser::ParseNamespace(unsigned Context,
     
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteNamespaceDecl(getCurScope());
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
+    return 0;
   }
 
   SourceLocation IdentLoc;
@@ -224,7 +225,8 @@ Decl *Parser::ParseNamespaceAlias(SourceLocation NamespaceLoc,
 
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteNamespaceAliasDecl(getCurScope());
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
+    return 0;
   }
 
   CXXScopeSpec SS;
@@ -324,7 +326,8 @@ Decl *Parser::ParseUsingDirectiveOrDeclaration(unsigned Context,
 
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteUsing(getCurScope());
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
+    return 0;
   }
 
   // 'using namespace' means this is a using-directive.
@@ -369,7 +372,8 @@ Decl *Parser::ParseUsingDirective(unsigned Context,
 
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteUsingDirective(getCurScope());
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
+    return 0;
   }
 
   CXXScopeSpec SS;
@@ -852,7 +856,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
   if (Tok.is(tok::code_completion)) {
     // Code completion for a struct, class, or union name.
     Actions.CodeCompleteTag(getCurScope(), TagType);
-    ConsumeCodeCompletionToken();
+    return cutOffParsing();
   }
 
   // C++03 [temp.explicit] 14.7.2/8:
@@ -2202,7 +2206,7 @@ void Parser::ParseConstructorInitializer(Decl *ConstructorDecl) {
       Actions.CodeCompleteConstructorInitializer(ConstructorDecl, 
                                                  MemInitializers.data(), 
                                                  MemInitializers.size());
-      ConsumeCodeCompletionToken();
+      return cutOffParsing();
     } else {
       MemInitResult MemInit = ParseMemInitializer(ConstructorDecl);
       if (!MemInit.isInvalid())

clang/lib/Parse/ParseExpr.cpp

@@ -214,7 +214,8 @@ Parser::ParseExpressionWithLeadingExtension(SourceLocation ExtLoc) {
 ExprResult Parser::ParseAssignmentExpression() {
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Expression);
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
+    return ExprError();
   }
 
   if (Tok.is(tok::kw_throw))
@@ -336,7 +337,7 @@ Parser::ParseRHSOfBinaryExpression(ExprResult LHS, prec::Level MinPrec) {
     // goes through a special hook that takes the left-hand side into account.
     if (Tok.is(tok::code_completion) && NextTokPrec == prec::Assignment) {
       Actions.CodeCompleteAssignmentRHS(getCurScope(), LHS.get());
-      ConsumeCodeCompletionToken();
+      cutOffParsing();
       return ExprError();
     }
     
@@ -1110,9 +1111,8 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
     break;
   case tok::code_completion: {
     Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Expression);
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
-    return ParseCastExpression(isUnaryExpression, isAddressOfOperand, 
+    return ExprError();
-                               NotCastExpr, isTypeCast);
   }
   case tok::l_square:
     if (getLang().CPlusPlus0x) {
@@ -1170,9 +1170,8 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
         return move(LHS);
         
       Actions.CodeCompletePostfixExpression(getCurScope(), LHS);
-      ConsumeCodeCompletionToken();
+      cutOffParsing();
-      LHS = ExprError();
+      return ExprError();
-      break;
         
     case tok::identifier:
       // If we see identifier: after an expression, and we're not already in a
@@ -1272,7 +1271,8 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
       
       if (Tok.is(tok::code_completion)) {
         Actions.CodeCompleteCall(getCurScope(), LHS.get(), 0, 0);
-        ConsumeCodeCompletionToken();
+        cutOffParsing();
+        return ExprError();
       }
 
       if (OpKind == tok::l_paren || !LHS.isInvalid()) {
@@ -1330,7 +1330,8 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
         Actions.CodeCompleteMemberReferenceExpr(getCurScope(), LHS.get(),
                                                 OpLoc, OpKind == tok::arrow);
         
-        ConsumeCodeCompletionToken();
+        cutOffParsing();
+        return ExprError();
       }
       
       if (MayBePseudoDestructor && !LHS.isInvalid()) {
@@ -1778,7 +1779,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
     Actions.CodeCompleteOrdinaryName(getCurScope(), 
                  ExprType >= CompoundLiteral? Sema::PCC_ParenthesizedExpression
                                             : Sema::PCC_Expression);
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
     return ExprError();
   }
 
@@ -2133,7 +2134,8 @@ bool Parser::ParseExpressionList(SmallVectorImpl<Expr*> &Exprs,
         (Actions.*Completer)(getCurScope(), Data, Exprs.data(), Exprs.size());
       else
         Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Expression);
-      ConsumeCodeCompletionToken();
+      cutOffParsing();
+      return true;
     }
 
     ExprResult Expr;
@@ -2164,7 +2166,7 @@ bool Parser::ParseExpressionList(SmallVectorImpl<Expr*> &Exprs,
 void Parser::ParseBlockId() {
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Type);
-    ConsumeCodeCompletionToken();
+    return cutOffParsing();
   }
   
   // Parse the specifier-qualifier-list piece.

clang/lib/Parse/ParseExprCXX.cpp

@@ -162,12 +162,13 @@ bool Parser::ParseOptionalCXXScopeSpecifier(CXXScopeSpec &SS,
         // Code completion for a nested-name-specifier, where the code
         // code completion token follows the '::'.
         Actions.CodeCompleteQualifiedId(getCurScope(), SS, EnteringContext);
-        SourceLocation ccLoc = ConsumeCodeCompletionToken();
         // Include code completion token into the range of the scope otherwise
         // when we try to annotate the scope tokens the dangling code completion
         // token will cause assertion in
         // Preprocessor::AnnotatePreviousCachedTokens.
-        SS.setEndLoc(ccLoc);
+        SS.setEndLoc(Tok.getLocation());
+        cutOffParsing();
+        return true;
       }
     }
 
@@ -1150,7 +1151,8 @@ bool Parser::ParseCXXCondition(ExprResult &ExprOut,
                                bool ConvertToBoolean) {
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Condition);
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
+    return true;
   }
 
   if (!isCXXConditionDeclaration()) {
@@ -1713,10 +1715,7 @@ bool Parser::ParseUnqualifiedIdOperator(CXXScopeSpec &SS, bool EnteringContext,
     case tok::code_completion: {
       // Code completion for the operator name.
       Actions.CodeCompleteOperatorName(getCurScope());
-      
+      cutOffParsing();      
-      // Consume the operator token.
-      ConsumeCodeCompletionToken();
-      
       // Don't try to parse any further.
       return true;
     }

clang/lib/Parse/ParseObjc.cpp

@@ -34,7 +34,8 @@ Parser::DeclGroupPtrTy Parser::ParseObjCAtDirectives() {
 
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteObjCAtDirective(getCurScope());
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
+    return DeclGroupPtrTy();
   }
     
   Decl *SingleDecl = 0;
@@ -149,7 +150,8 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation atLoc,
   // Code completion after '@interface'.
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteObjCInterfaceDecl(getCurScope());
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
+    return 0;
   }
 
   if (Tok.isNot(tok::identifier)) {
@@ -169,7 +171,8 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation atLoc,
     IdentifierInfo *categoryId = 0;
     if (Tok.is(tok::code_completion)) {
       Actions.CodeCompleteObjCInterfaceCategory(getCurScope(), nameId, nameLoc);
-      ConsumeCodeCompletionToken();
+      cutOffParsing();
+      return 0;
     }
     
     // For ObjC2, the category name is optional (not an error).
@@ -224,7 +227,8 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation atLoc,
     // Code completion of superclass names.
     if (Tok.is(tok::code_completion)) {
       Actions.CodeCompleteObjCSuperclass(getCurScope(), nameId, nameLoc);
-      ConsumeCodeCompletionToken();
+      cutOffParsing();
+      return 0;
     }
 
     if (Tok.isNot(tok::identifier)) {
@@ -369,7 +373,7 @@ void Parser::ParseObjCInterfaceDeclList(tok::ObjCKeywordKind contextKey,
       Actions.CodeCompleteOrdinaryName(getCurScope(), 
                                   ObjCImpDecl? Sema::PCC_ObjCImplementation
                                              : Sema::PCC_ObjCInterface);
-      ConsumeCodeCompletionToken();
+      return cutOffParsing();
     }
     
     // If we don't have an @ directive, parse it as a function definition.
@@ -388,7 +392,7 @@ void Parser::ParseObjCInterfaceDeclList(tok::ObjCKeywordKind contextKey,
     SourceLocation AtLoc = ConsumeToken(); // the "@"
     if (Tok.is(tok::code_completion)) {
       Actions.CodeCompleteObjCAtDirective(getCurScope());
-      ConsumeCodeCompletionToken();
+      return cutOffParsing();
       break;
     }
 
@@ -459,7 +463,7 @@ void Parser::ParseObjCInterfaceDeclList(tok::ObjCKeywordKind contextKey,
   // EOF.  In the former case, eat the @end.  In the later case, emit an error.
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteObjCAtDirective(getCurScope());
-    ConsumeCodeCompletionToken();
+    return cutOffParsing();
   } else if (Tok.isObjCAtKeyword(tok::objc_end))
     ConsumeToken(); // the "end" identifier
   else
@@ -500,7 +504,7 @@ void Parser::ParseObjCPropertyAttribute(ObjCDeclSpec &DS) {
   while (1) {
     if (Tok.is(tok::code_completion)) {
       Actions.CodeCompleteObjCPropertyFlags(getCurScope(), DS);
-      ConsumeCodeCompletionToken();
+      return cutOffParsing();
     }
     const IdentifierInfo *II = Tok.getIdentifierInfo();
 
@@ -547,7 +551,7 @@ void Parser::ParseObjCPropertyAttribute(ObjCDeclSpec &DS) {
           Actions.CodeCompleteObjCPropertySetter(getCurScope());
         else
           Actions.CodeCompleteObjCPropertyGetter(getCurScope());
-        ConsumeCodeCompletionToken();
+        return cutOffParsing();
       }
 
       
@@ -744,7 +748,7 @@ void Parser::ParseObjCTypeQualifierList(ObjCDeclSpec &DS,
     if (Tok.is(tok::code_completion)) {
       Actions.CodeCompleteObjCPassingType(getCurScope(), DS, 
                                           Context == OTN_ParameterType);
-      ConsumeCodeCompletionToken();
+      return cutOffParsing();
     }
     
     if (Tok.isNot(tok::identifier))
@@ -850,7 +854,8 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteObjCMethodDecl(getCurScope(), mType == tok::minus, 
                                        /*ReturnType=*/ ParsedType());
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
+    return 0;
   }
 
   // Parse the return type if present.
@@ -867,7 +872,8 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteObjCMethodDecl(getCurScope(), mType == tok::minus, 
                                        ReturnType);
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
+    return 0;
   }
 
   // Now parse the selector.
@@ -932,7 +938,6 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
 
     // Code completion for the next piece of the selector.
     if (Tok.is(tok::code_completion)) {
-      ConsumeCodeCompletionToken();
       KeyIdents.push_back(SelIdent);
       Actions.CodeCompleteObjCMethodDeclSelector(getCurScope(), 
                                                  mType == tok::minus,
@@ -940,8 +945,8 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
                                                  ReturnType,
                                                  KeyIdents.data(), 
                                                  KeyIdents.size());
-      KeyIdents.pop_back();
+      cutOffParsing();
-      break;
+      return 0;
     }
     
     if (Tok.isNot(tok::identifier)) {
@@ -961,14 +966,14 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
 
     // Code completion for the next piece of the selector.
     if (Tok.is(tok::code_completion)) {
-      ConsumeCodeCompletionToken();
       Actions.CodeCompleteObjCMethodDeclSelector(getCurScope(), 
                                                  mType == tok::minus,
                                                  /*AtParameterName=*/false,
                                                  ReturnType,
                                                  KeyIdents.data(), 
                                                  KeyIdents.size());
-      break;
+      cutOffParsing();
+      return 0;
     }
     
     // Check for another keyword selector.
@@ -1043,7 +1048,8 @@ ParseObjCProtocolReferences(SmallVectorImpl<Decl *> &Protocols,
     if (Tok.is(tok::code_completion)) {
       Actions.CodeCompleteObjCProtocolReferences(ProtocolIdents.data(), 
                                                  ProtocolIdents.size());
-      ConsumeCodeCompletionToken();
+      cutOffParsing();
+      return true;
     }
 
     if (Tok.isNot(tok::identifier)) {
@@ -1142,7 +1148,7 @@ void Parser::ParseObjCClassInstanceVariables(Decl *interfaceDecl,
       
       if (Tok.is(tok::code_completion)) {
         Actions.CodeCompleteObjCAtVisibility(getCurScope());
-        ConsumeCodeCompletionToken();
+        return cutOffParsing();
       }
       
       switch (Tok.getObjCKeywordID()) {
@@ -1162,7 +1168,7 @@ void Parser::ParseObjCClassInstanceVariables(Decl *interfaceDecl,
     if (Tok.is(tok::code_completion)) {
       Actions.CodeCompleteOrdinaryName(getCurScope(), 
                                        Sema::PCC_ObjCInstanceVariableList);
-      ConsumeCodeCompletionToken();
+      return cutOffParsing();
     }
     
     struct ObjCIvarCallback : FieldCallback {
@@ -1238,7 +1244,8 @@ Decl *Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
 
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteObjCProtocolDecl(getCurScope());
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
+    return 0;
   }
 
   if (Tok.isNot(tok::identifier)) {
@@ -1325,7 +1332,8 @@ Decl *Parser::ParseObjCAtImplementationDeclaration(
   // Code completion after '@implementation'.
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteObjCImplementationDecl(getCurScope());
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
+    return 0;
   }
 
   if (Tok.isNot(tok::identifier)) {
@@ -1344,7 +1352,8 @@ Decl *Parser::ParseObjCAtImplementationDeclaration(
 
     if (Tok.is(tok::code_completion)) {
       Actions.CodeCompleteObjCImplementationCategory(getCurScope(), nameId, nameLoc);
-      ConsumeCodeCompletionToken();
+      cutOffParsing();
+      return 0;
     }
     
     if (Tok.is(tok::identifier)) {
@@ -1475,7 +1484,8 @@ Decl *Parser::ParseObjCPropertySynthesize(SourceLocation atLoc) {
   while (true) {
     if (Tok.is(tok::code_completion)) {
       Actions.CodeCompleteObjCPropertyDefinition(getCurScope());
-      ConsumeCodeCompletionToken();
+      cutOffParsing();
+      return 0;
     }
     
     if (Tok.isNot(tok::identifier)) {
@@ -1494,7 +1504,8 @@ Decl *Parser::ParseObjCPropertySynthesize(SourceLocation atLoc) {
       
       if (Tok.is(tok::code_completion)) {
         Actions.CodeCompleteObjCPropertySynthesizeIvar(getCurScope(), propertyId);
-        ConsumeCodeCompletionToken();
+        cutOffParsing();
+        return 0;
       }
       
       if (Tok.isNot(tok::identifier)) {
@@ -1528,7 +1539,8 @@ Decl *Parser::ParseObjCPropertyDynamic(SourceLocation atLoc) {
   while (true) {
     if (Tok.is(tok::code_completion)) {
       Actions.CodeCompleteObjCPropertyDefinition(getCurScope());
-      ConsumeCodeCompletionToken();
+      cutOffParsing();
+      return 0;
     }
     
     if (Tok.isNot(tok::identifier)) {
@@ -1801,7 +1813,7 @@ Decl *Parser::ParseObjCMethodDefinition() {
 StmtResult Parser::ParseObjCAtStatement(SourceLocation AtLoc) {
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteObjCAtStatement(getCurScope());
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
     return StmtError();
   }
   
@@ -1835,7 +1847,7 @@ ExprResult Parser::ParseObjCAtExpression(SourceLocation AtLoc) {
   switch (Tok.getKind()) {
   case tok::code_completion:
     Actions.CodeCompleteObjCAtExpression(getCurScope());
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
     return ExprError();
 
   case tok::string_literal:    // primary-expression: string-literal
@@ -2001,8 +2013,7 @@ ExprResult Parser::ParseObjCMessageExpression() {
 
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteObjCMessageReceiver(getCurScope());
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
-    SkipUntil(tok::r_square);
     return ExprError();
   }
   
@@ -2133,7 +2144,8 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
     else
       Actions.CodeCompleteObjCInstanceMessage(getCurScope(), ReceiverExpr,
                                               0, 0, false);
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
+    return ExprError();
   }
   
   // Parse objc-selector
@@ -2179,8 +2191,7 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
                                                   KeyIdents.size(),
                                                   /*AtArgumentEpression=*/true);
 
-        ConsumeCodeCompletionToken();
+        cutOffParsing();
-        SkipUntil(tok::r_square);
         return ExprError();
       }
       
@@ -2213,8 +2224,7 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
                                                   KeyIdents.data(), 
                                                   KeyIdents.size(),
                                                 /*AtArgumentEpression=*/false);
-        ConsumeCodeCompletionToken();
+        cutOffParsing();
-        SkipUntil(tok::r_square);
         return ExprError();
       }
             
@@ -2380,8 +2390,7 @@ ExprResult Parser::ParseObjCSelectorExpression(SourceLocation AtLoc) {
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteObjCSelector(getCurScope(), KeyIdents.data(),
                                      KeyIdents.size());
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
-    MatchRHSPunctuation(tok::r_paren, LParenLoc);
     return ExprError();
   }
   
@@ -2408,8 +2417,7 @@ ExprResult Parser::ParseObjCSelectorExpression(SourceLocation AtLoc) {
       if (Tok.is(tok::code_completion)) {
         Actions.CodeCompleteObjCSelector(getCurScope(), KeyIdents.data(),
                                          KeyIdents.size());
-        ConsumeCodeCompletionToken();
+        cutOffParsing();
-        MatchRHSPunctuation(tok::r_paren, LParenLoc);
         return ExprError();
       }
 

clang/lib/Parse/ParseStmt.cpp

@@ -100,8 +100,8 @@ Retry:
 
   case tok::code_completion:
     Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Statement);
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
-    return ParseStatementOrDeclaration(Stmts, OnlyStatement);
+    return StmtError();
       
   case tok::identifier: {
     Token Next = NextToken();
@@ -507,7 +507,8 @@ StmtResult Parser::ParseCaseStatement(ParsedAttributes &attrs, bool MissingCase,
 
     if (Tok.is(tok::code_completion)) {
       Actions.CodeCompleteCase(getCurScope());
-      ConsumeCodeCompletionToken();
+      cutOffParsing();
+      return StmtError();
     }
     
     /// We don't want to treat 'case x : y' as a potential typo for 'case x::y'.
@@ -953,7 +954,8 @@ StmtResult Parser::ParseIfStatement(ParsedAttributes &attrs) {
     InnerScope.Exit();
   } else if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteAfterIf(getCurScope());
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
+    return StmtError();
   }
 
   IfScope.Exit();
@@ -1282,7 +1284,8 @@ StmtResult Parser::ParseForStatement(ParsedAttributes &attrs) {
     Actions.CodeCompleteOrdinaryName(getCurScope(), 
                                      C99orCXXorObjC? Sema::PCC_ForInit
                                                    : Sema::PCC_Expression);
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
+    return StmtError();
   }
   
   // Parse the first part of the for specifier.
@@ -1320,7 +1323,8 @@ StmtResult Parser::ParseForStatement(ParsedAttributes &attrs) {
       
       if (Tok.is(tok::code_completion)) {
         Actions.CodeCompleteObjCForCollection(getCurScope(), DG);
-        ConsumeCodeCompletionToken();
+        cutOffParsing();
+        return StmtError();
       }
       Collection = ParseExpression();
     } else {
@@ -1346,7 +1350,8 @@ StmtResult Parser::ParseForStatement(ParsedAttributes &attrs) {
       
       if (Tok.is(tok::code_completion)) {
         Actions.CodeCompleteObjCForCollection(getCurScope(), DeclGroupPtrTy());
-        ConsumeCodeCompletionToken();
+        cutOffParsing();
+        return StmtError();
       }
       Collection = ParseExpression();
     } else {
@@ -1537,8 +1542,7 @@ StmtResult Parser::ParseReturnStatement(ParsedAttributes &attrs) {
   if (Tok.isNot(tok::semi)) {
     if (Tok.is(tok::code_completion)) {
       Actions.CodeCompleteReturn(getCurScope());
-      ConsumeCodeCompletionToken();
+      cutOffParsing();
-      SkipUntil(tok::semi, false, true);
       return StmtError();
     }
         

clang/lib/Parse/Parser.cpp

@@ -554,6 +554,11 @@ Parser::ParseExternalDeclaration(ParsedAttributesWithRange &attrs,
   DelayedCleanupPoint CleanupRAII(TopLevelDeclCleanupPool);
   ParenBraceBracketBalancer BalancerRAIIObj(*this);
 
+  if (PP.isCodeCompletionReached()) {
+    cutOffParsing();
+    return DeclGroupPtrTy();
+  }
+
   Decl *SingleDecl = 0;
   switch (Tok.getKind()) {
   case tok::semi:
@@ -608,8 +613,8 @@ Parser::ParseExternalDeclaration(ParsedAttributesWithRange &attrs,
       Actions.CodeCompleteOrdinaryName(getCurScope(), 
                                    ObjCImpDecl? Sema::PCC_ObjCImplementation
                                               : Sema::PCC_Namespace);
-    ConsumeCodeCompletionToken();
+    cutOffParsing();
-    return ParseExternalDeclaration(attrs);
+    return DeclGroupPtrTy();
   case tok::kw_using:
   case tok::kw_namespace:
   case tok::kw_typedef:
@@ -1413,20 +1418,27 @@ bool Parser::isTokenEqualOrMistypedEqualEqual(unsigned DiagID) {
   return Tok.is(tok::equal);
 }
 
-void Parser::CodeCompletionRecovery() {
+SourceLocation Parser::handleUnexpectedCodeCompletionToken() {
+  assert(Tok.is(tok::code_completion));
+  PrevTokLocation = Tok.getLocation();
+
   for (Scope *S = getCurScope(); S; S = S->getParent()) {
     if (S->getFlags() & Scope::FnScope) {
       Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_RecoveryInFunction);
-      return;
+      cutOffParsing();
+      return PrevTokLocation;
     }
     
     if (S->getFlags() & Scope::ClassScope) {
       Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Class);
-      return;
+      cutOffParsing();
+      return PrevTokLocation;
     }
   }
   
   Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Namespace);
+  cutOffParsing();
+  return PrevTokLocation;
 }
 
 // Anchor the Parser::FieldCallback vtable to this translation unit.

clang/test/CodeCompletion/truncation.c

@@ -1,18 +1,15 @@
 #include "truncation.c.h"
 
-struct 
-
 /* foo */
 
+struct 
+
 // RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s.h:4:8 -o - %s | FileCheck -check-prefix=CC1 %s
 // CHECK-CC1: X
 // CHECK-CC1-NEXT: Y
-// RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:3:8 -o - %s | FileCheck -check-prefix=CC2 %s
+// RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:5:8 -o - %s | FileCheck -check-prefix=CC2 %s
 // CHECK-CC2: X
 // CHECK-CC2: Xa
 // CHECK-CC2: Y
 
-// RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:5:3 -o - %s | FileCheck -check-prefix=CC3 %s
+// RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:3:3 -o - %s
-// CHECK-CC3: X
-// CHECK-CC3: Xa
-// CHECK-CC3: Y

clang/test/Index/complete-at-exprstmt.m

@@ -1,6 +1,5 @@
 /* The run lines are below, because this test is line- and
    column-number sensitive. */
-// XFAIL: *
 @interface MyClass { int ivar; }
 - (int)myMethod:(int)arg;
 @end

clang/test/Index/complete-cxx-inline-methods.cpp

@@ -0,0 +1,24 @@
+class MyCls {
+  void in_foo() {
+    vec.x = 0;
+  }
+  void out_foo();
+
+  struct Vec { int x, y; };
+  Vec vec;
+};
+
+void MyCls::out_foo() {
+  vec.x = 0;
+}
+
+// RUN: c-index-test -code-completion-at=%s:3:9 %s | FileCheck %s
+// RUN: c-index-test -code-completion-at=%s:12:7 %s | FileCheck %s
+// CHECK:      CXXMethod:{ResultType MyCls::Vec &}{TypedText operator=}{LeftParen (}{Placeholder const MyCls::Vec &}{RightParen )} (34)
+// CHECK-NEXT: StructDecl:{TypedText Vec}{Text ::} (75)
+// CHECK-NEXT: FieldDecl:{ResultType int}{TypedText x} (35)
+// CHECK-NEXT: FieldDecl:{ResultType int}{TypedText y} (35)
+// CHECK-NEXT: CXXDestructor:{ResultType void}{TypedText ~Vec}{LeftParen (}{RightParen )} (34)
+// CHECK-NEXT: Completion contexts:
+// CHECK-NEXT: Dot member access
+// CHECK-NEXT: Container Kind: StructDecl

clang/test/Index/complete-declarators.m

@@ -1,5 +1,4 @@
 // This test is line- and column-sensitive, so test commands are at the bottom.
-// XFAIL: *
 @protocol P
 - (int)method:(id)param1;
 @end

clang/test/Index/complete-exprs.m

@@ -1,4 +1,3 @@
-// XFAIL: *
 typedef signed char BOOL;
 #define YES ((BOOL)1)
 #define NO ((BOOL)0)

clang/test/Index/complete-objc-message-id.m

@@ -1,7 +1,6 @@
 // Note: the run lines follow their respective tests, since line/column
 // matter in this test.
 
-// XFAIL: *
 @interface A
 + (id)alloc;
 + (id)init;

clang/test/Index/complete-objc-message.m

@@ -1,6 +1,5 @@
 // Note: the run lines follow their respective tests, since line/column
 // matter in this test.
-// XFAIL: *
 #define nil (void*)0
 @protocol FooTestProtocol
 + protocolClassMethod;

clang/test/Index/complete-recovery.m

@@ -1,6 +1,5 @@
 /* Run lines are at the end, since line/column matter in this test. */
 
-// XFAIL: *
 @interface A
 - (void)method:(int)x;
 @end

clang/test/Index/complete-super.m

@@ -1,6 +1,5 @@
 // Note: the run lines follow their respective tests, since line/column
 // matter in this test.
-// XFAIL: *
 
 typedef int Bool;
 

clang/test/Index/complete-synthesized.m

@@ -1,6 +1,5 @@
 // Note: this test is line- and column-sensitive. Test commands are at
 // the end.
-// XFAIL: *
 
 
 @interface A
@@ -36,22 +35,12 @@
 }
 @end
 
-// RUN: c-index-test -code-completion-at=%s:24:1 -Xclang -fobjc-nonfragile-abi %s | FileCheck -check-prefix=CHECK-CC1 %s
+// RUN: c-index-test -code-completion-at=%s:24:1 -Xclang -fobjc-nonfragile-abi -fobjc-default-synthesize-properties %s | FileCheck %s
-// CHECK-CC1: NotImplemented:{TypedText _Bool} (50)
+// RUN: c-index-test -code-completion-at=%s:30:2 -Xclang -fobjc-nonfragile-abi -fobjc-default-synthesize-properties %s | FileCheck %s
-// CHECK-CC1: ObjCIvarDecl:{ResultType float}{TypedText _prop2} (35)
+// RUN: c-index-test -code-completion-at=%s:34:2 -Xclang -fobjc-nonfragile-abi -fobjc-default-synthesize-properties %s | FileCheck %s
-// CHECK-CC1-NOT: prop2
-// CHECK-CC1: ObjCPropertyDecl:{ResultType short}{TypedText prop3} (35)
-// CHECK-CC1: ObjCPropertyDecl:{ResultType double}{TypedText prop4} (35)
 
-// RUN: c-index-test -code-completion-at=%s:30:2 -Xclang -fobjc-nonfragile-abi %s | FileCheck -check-prefix=CHECK-CC2 %s
+// CHECK: NotImplemented:{TypedText _Bool} (50)
-// CHECK-CC2: NotImplemented:{TypedText _Bool} (50)
+// CHECK: ObjCIvarDecl:{ResultType float}{TypedText _prop2} (35)
-// CHECK-CC2: ObjCIvarDecl:{ResultType float}{TypedText _prop2} (35)
+// CHECK-NOT: prop2
-// CHECK-CC2-NOT: prop3
+// CHECK-NOT: prop3
-// CHECK-CC2: ObjCPropertyDecl:{ResultType double}{TypedText prop4} (35)
+// CHECK: ObjCIvarDecl:{ResultType double}{TypedText prop4} (37)
-
-// RUN: c-index-test -code-completion-at=%s:34:2 -Xclang -fobjc-nonfragile-abi %s | FileCheck -check-prefix=CHECK-CC3 %s
-// CHECK-CC3: NotImplemented:{TypedText _Bool} (50)
-// CHECK-CC3: ObjCIvarDecl:{ResultType float}{TypedText _prop2} (35)
-// CHECK-CC3: ObjCPropertyDecl:{ResultType double}{TypedText prop4}
-// CHECK-CC3-NOT: ObjCPropertyDecl:{ResultType double}{TypedText prop4} (35)
-// CHECK-CC1: restrict