[clangd] Add code completion support

Summary: Adds code completion support to clangd.

Reviewers: bkramer, malaperle-ericsson

Reviewed By: bkramer, malaperle-ericsson

Subscribers: stanionascu, malaperle-ericsson, cfe-commits

Differential Revision: https://reviews.llvm.org/D31328

llvm-svn: 299421

clang-tools-extra/clangd/ASTManager.cpp

@@ -83,50 +83,54 @@ void ASTManager::runWorker() {
 }
 
 void ASTManager::parseFileAndPublishDiagnostics(StringRef File) {
-  auto &Unit = ASTs[File]; // Only one thread can access this at a time.
-
-  if (!Unit) {
-    Unit = createASTUnitForFile(File, this->Store);
-  } else {
-    // Do a reparse if this wasn't the first parse.
-    // FIXME: This might have the wrong working directory if it changed in the
-    // meantime.
-    Unit->Reparse(PCHs, getRemappedFiles(this->Store));
-  }
-
-  if (!Unit)
-    return;
-
-  // Send the diagnotics to the editor.
-  // FIXME: If the diagnostic comes from a different file, do we want to
-  // show them all? Right now we drop everything not coming from the
-  // main file.
-  std::string Diagnostics;
   DiagnosticToReplacementMap LocalFixIts; // Temporary storage
-  for (ASTUnit::stored_diag_iterator D = Unit->stored_diag_begin(),
-                                     DEnd = Unit->stored_diag_end();
-       D != DEnd; ++D) {
-    if (!D->getLocation().isValid() ||
-        !D->getLocation().getManager().isInMainFile(D->getLocation()))
-      continue;
-    Position P;
-    P.line = D->getLocation().getSpellingLineNumber() - 1;
-    P.character = D->getLocation().getSpellingColumnNumber();
-    Range R = {P, P};
-    Diagnostics +=
-        R"({"range":)" + Range::unparse(R) +
-        R"(,"severity":)" + std::to_string(getSeverity(D->getLevel())) +
-        R"(,"message":")" + llvm::yaml::escape(D->getMessage()) +
-        R"("},)";
+  std::string Diagnostics;
+  {
+    std::lock_guard<std::mutex> ASTGuard(ASTLock);
+    auto &Unit = ASTs[File]; // Only one thread can access this at a time.
 
-    // We convert to Replacements to become independent of the SourceManager.
-    clangd::Diagnostic Diag = {R, getSeverity(D->getLevel()), D->getMessage()};
-    auto &FixItsForDiagnostic = LocalFixIts[Diag];
-    for (const FixItHint &Fix : D->getFixIts()) {
-      FixItsForDiagnostic.push_back(clang::tooling::Replacement(
-          Unit->getSourceManager(), Fix.RemoveRange, Fix.CodeToInsert));
+    if (!Unit) {
+      Unit = createASTUnitForFile(File, this->Store);
+    } else {
+      // Do a reparse if this wasn't the first parse.
+      // FIXME: This might have the wrong working directory if it changed in the
+      // meantime.
+      Unit->Reparse(PCHs, getRemappedFiles(this->Store));
     }
-  }
+
+    if (!Unit)
+      return;
+
+    // Send the diagnotics to the editor.
+    // FIXME: If the diagnostic comes from a different file, do we want to
+    // show them all? Right now we drop everything not coming from the
+    // main file.
+    for (ASTUnit::stored_diag_iterator D = Unit->stored_diag_begin(),
+                                       DEnd = Unit->stored_diag_end();
+         D != DEnd; ++D) {
+      if (!D->getLocation().isValid() ||
+          !D->getLocation().getManager().isInMainFile(D->getLocation()))
+        continue;
+      Position P;
+      P.line = D->getLocation().getSpellingLineNumber() - 1;
+      P.character = D->getLocation().getSpellingColumnNumber();
+      Range R = {P, P};
+      Diagnostics +=
+          R"({"range":)" + Range::unparse(R) +
+          R"(,"severity":)" + std::to_string(getSeverity(D->getLevel())) +
+          R"(,"message":")" + llvm::yaml::escape(D->getMessage()) +
+          R"("},)";
+
+      // We convert to Replacements to become independent of the SourceManager.
+      clangd::Diagnostic Diag = {R, getSeverity(D->getLevel()),
+                                 D->getMessage()};
+      auto &FixItsForDiagnostic = LocalFixIts[Diag];
+      for (const FixItHint &Fix : D->getFixIts()) {
+        FixItsForDiagnostic.push_back(clang::tooling::Replacement(
+            Unit->getSourceManager(), Fix.RemoveRange, Fix.CodeToInsert));
+      }
+    }
+  } // unlock ASTLock
 
   // Put FixIts into place.
   {
@@ -232,3 +236,79 @@ ASTManager::getFixIts(const clangd::Diagnostic &D) {
     return I->second;
   return {};
 }
+
+namespace {
+
+class CompletionItemsCollector : public CodeCompleteConsumer {
+  std::vector<CompletionItem> *Items;
+  std::shared_ptr<clang::GlobalCodeCompletionAllocator> Allocator;
+  CodeCompletionTUInfo CCTUInfo;
+
+public:
+  CompletionItemsCollector(std::vector<CompletionItem> *Items,
+                           const CodeCompleteOptions &CodeCompleteOpts)
+      : CodeCompleteConsumer(CodeCompleteOpts, /*OutputIsBinary=*/false),
+        Items(Items),
+        Allocator(std::make_shared<clang::GlobalCodeCompletionAllocator>()),
+        CCTUInfo(Allocator) {}
+
+  void ProcessCodeCompleteResults(Sema &S, CodeCompletionContext Context,
+                                  CodeCompletionResult *Results,
+                                  unsigned NumResults) override {
+    for (unsigned I = 0; I != NumResults; ++I) {
+      CodeCompletionString *CCS = Results[I].CreateCodeCompletionString(
+          S, Context, *Allocator, CCTUInfo,
+          CodeCompleteOpts.IncludeBriefComments);
+      if (CCS) {
+        CompletionItem Item;
+        assert(CCS->getTypedText());
+        Item.label = CCS->getTypedText();
+        if (CCS->getBriefComment())
+          Item.documentation = CCS->getBriefComment();
+        Items->push_back(std::move(Item));
+      }
+    }
+  }
+
+  GlobalCodeCompletionAllocator &getAllocator() override { return *Allocator; }
+
+  CodeCompletionTUInfo &getCodeCompletionTUInfo() override { return CCTUInfo; }
+};
+
+} // namespace
+
+std::vector<CompletionItem>
+ASTManager::codeComplete(StringRef Uri, unsigned Line, unsigned Column) {
+  CodeCompleteOptions CCO;
+  CCO.IncludeBriefComments = 1;
+  // This is where code completion stores dirty buffers. Need to free after
+  // completion.
+  SmallVector<const llvm::MemoryBuffer *, 4> OwnedBuffers;
+  SmallVector<StoredDiagnostic, 4> StoredDiagnostics;
+  IntrusiveRefCntPtr<DiagnosticsEngine> DiagEngine(
+      new DiagnosticsEngine(new DiagnosticIDs, new DiagnosticOptions));
+  std::vector<CompletionItem> Items;
+  CompletionItemsCollector Collector(&Items, CCO);
+  std::lock_guard<std::mutex> Guard(ASTLock);
+  auto &Unit = ASTs[Uri];
+  if (!Unit)
+    Unit = createASTUnitForFile(Uri, this->Store);
+  if (!Unit)
+    return {};
+  IntrusiveRefCntPtr<SourceManager> SourceMgr(
+      new SourceManager(*DiagEngine, Unit->getFileManager()));
+  StringRef File(Uri);
+  File.consume_front("file://");
+  // CodeComplete seems to require fresh LangOptions.
+  LangOptions LangOpts = Unit->getLangOpts();
+  // The language server protocol uses zero-based line and column numbers.
+  // The clang code completion uses one-based numbers.
+  Unit->CodeComplete(File, Line + 1, Column + 1, getRemappedFiles(this->Store),
+                     CCO.IncludeMacros, CCO.IncludeCodePatterns,
+                     CCO.IncludeBriefComments, Collector, PCHs, *DiagEngine,
+                     LangOpts, *SourceMgr, Unit->getFileManager(),
+                     StoredDiagnostics, OwnedBuffers);
+  for (const llvm::MemoryBuffer *Buffer : OwnedBuffers)
+    delete Buffer;
+  return Items;
+}

clang-tools-extra/clangd/ASTManager.h

@@ -36,7 +36,13 @@ public:
 
   void onDocumentAdd(StringRef Uri) override;
   // FIXME: Implement onDocumentRemove
-  // FIXME: Implement codeComplete
+
+  /// Get code completions at a specified \p Line and \p Column in \p File.
+  ///
+  /// This function is thread-safe and returns completion items that own the
+  /// data they contain.
+  std::vector<CompletionItem> codeComplete(StringRef File, unsigned Line,
+                                           unsigned Column);
 
   /// Get the fixes associated with a certain diagnostic as replacements.
   ///
@@ -59,7 +65,7 @@ private:
   /// database is cached for subsequent accesses.
   clang::tooling::CompilationDatabase *
   getOrCreateCompilationDatabaseForFile(StringRef Uri);
-  // Craetes a new ASTUnit for the document at Uri.
+  // Creates a new ASTUnit for the document at Uri.
   // FIXME: This calls chdir internally, which is thread unsafe.
   std::unique_ptr<clang::ASTUnit>
   createASTUnitForFile(StringRef Uri, const DocumentStore &Docs);
@@ -68,7 +74,17 @@ private:
   void parseFileAndPublishDiagnostics(StringRef File);
 
   /// Clang objects.
+
+  /// A map from Uri-s to ASTUnit-s. Guarded by \c ASTLock. ASTUnit-s are used
+  /// for generating diagnostics and fix-it-s asynchronously by the worker
+  /// thread and synchronously for code completion.
+  ///
+  /// TODO(krasimir): code completion should always have priority over parsing
+  /// for diagnostics.
   llvm::StringMap<std::unique_ptr<clang::ASTUnit>> ASTs;
+  /// A lock for access to the map \c ASTs.
+  std::mutex ASTLock;
+
   llvm::StringMap<std::unique_ptr<clang::tooling::CompilationDatabase>>
       CompilationDatabases;
   std::shared_ptr<clang::PCHContainerOperations> PCHs;

clang-tools-extra/clangd/ClangDMain.cpp

@@ -61,6 +61,8 @@ int main(int argc, char *argv[]) {
       llvm::make_unique<TextDocumentFormattingHandler>(Out, Store));
   Dispatcher.registerHandler("textDocument/codeAction",
                              llvm::make_unique<CodeActionHandler>(Out, AST));
+  Dispatcher.registerHandler("textDocument/completion",
+                             llvm::make_unique<CompletionHandler>(Out, AST));
 
   while (std::cin.good()) {
     // A Language Server Protocol message starts with a HTTP header, delimited

clang-tools-extra/clangd/Protocol.cpp

@@ -570,3 +570,75 @@ CodeActionParams::parse(llvm::yaml::MappingNode *Params) {
   }
   return Result;
 }
+
+llvm::Optional<TextDocumentPositionParams>
+TextDocumentPositionParams::parse(llvm::yaml::MappingNode *Params) {
+  TextDocumentPositionParams Result;
+  for (auto &NextKeyValue : *Params) {
+    auto *KeyString = dyn_cast<llvm::yaml::ScalarNode>(NextKeyValue.getKey());
+    if (!KeyString)
+      return llvm::None;
+
+    llvm::SmallString<10> KeyStorage;
+    StringRef KeyValue = KeyString->getValue(KeyStorage);
+    auto *Value =
+        dyn_cast_or_null<llvm::yaml::MappingNode>(NextKeyValue.getValue());
+    if (!Value)
+      return llvm::None;
+
+    llvm::SmallString<10> Storage;
+    if (KeyValue == "textDocument") {
+      auto Parsed = TextDocumentIdentifier::parse(Value);
+      if (!Parsed)
+        return llvm::None;
+      Result.textDocument = std::move(*Parsed);
+    } else if (KeyValue == "position") {
+      auto Parsed = Position::parse(Value);
+      if (!Parsed)
+        return llvm::None;
+      Result.position = std::move(*Parsed);
+    } else {
+      return llvm::None;
+    }
+  }
+  return Result;
+}
+
+std::string CompletionItem::unparse(const CompletionItem &CI) {
+  std::string Result = "{";
+  llvm::raw_string_ostream Os(Result);
+  assert(!CI.label.empty() && "completion item label is required");
+  Os << R"("label":")" << llvm::yaml::escape(CI.label) << R"(",)";
+  if (CI.kind != CompletionItemKind::Missing)
+    Os << R"("kind":)" << static_cast<int>(CI.kind) << R"(",)";
+  if (!CI.detail.empty())
+    Os << R"("detail":")" << llvm::yaml::escape(CI.detail) << R"(",)";
+  if (!CI.documentation.empty())
+    Os << R"("documentation":")" << llvm::yaml::escape(CI.documentation)
+       << R"(",)";
+  if (!CI.sortText.empty())
+    Os << R"("sortText":")" << llvm::yaml::escape(CI.sortText) << R"(",)";
+  if (!CI.filterText.empty())
+    Os << R"("filterText":")" << llvm::yaml::escape(CI.filterText) << R"(",)";
+  if (!CI.insertText.empty())
+    Os << R"("insertText":")" << llvm::yaml::escape(CI.insertText) << R"(",)";
+  if (CI.insertTextFormat != InsertTextFormat::Missing) {
+    Os << R"("insertTextFormat":")" << static_cast<int>(CI.insertTextFormat)
+       << R"(",)";
+  }
+  if (CI.textEdit)
+    Os << R"("textEdit":)" << TextEdit::unparse(*CI.textEdit) << ',';
+  if (!CI.additionalTextEdits.empty()) {
+    Os << R"("additionalTextEdits":[)";
+    for (const auto &Edit : CI.additionalTextEdits)
+      Os << TextEdit::unparse(Edit) << ",";
+    Os.flush();
+    // The list additionalTextEdits is guaranteed nonempty at this point.
+    // Replace the trailing comma with right brace.
+    Result.back() = ']';
+  }
+  Os.flush();
+  // Label is required, so Result is guaranteed to have a trailing comma.
+  Result.back() = '}';
+  return Result;
+}

clang-tools-extra/clangd/Protocol.h

@@ -233,6 +233,113 @@ struct CodeActionParams {
   parse(llvm::yaml::MappingNode *Params);
 };
 
+struct TextDocumentPositionParams {
+  /// The text document.
+  TextDocumentIdentifier textDocument;
+
+  /// The position inside the text document.
+  Position position;
+
+  static llvm::Optional<TextDocumentPositionParams>
+  parse(llvm::yaml::MappingNode *Params);
+};
+
+/// The kind of a completion entry.
+enum class CompletionItemKind {
+  Missing = 0,
+  Text = 1,
+  Method = 2,
+  Function = 3,
+  Constructor = 4,
+  Field = 5,
+  Variable = 6,
+  Class = 7,
+  Interface = 8,
+  Module = 9,
+  Property = 10,
+  Unit = 11,
+  Value = 12,
+  Enum = 13,
+  Keyword = 14,
+  Snippet = 15,
+  Color = 16,
+  File = 17,
+  Reference = 18,
+};
+
+/// Defines whether the insert text in a completion item should be interpreted
+/// as plain text or a snippet.
+enum class InsertTextFormat {
+  Missing = 0,
+  /// The primary text to be inserted is treated as a plain string.
+  PlainText = 1,
+  /// The primary text to be inserted is treated as a snippet.
+  ///
+  /// A snippet can define tab stops and placeholders with `$1`, `$2`
+  /// and `${3:foo}`. `$0` defines the final tab stop, it defaults to the end
+  /// of the snippet. Placeholders with equal identifiers are linked, that is
+  /// typing in one will update others too.
+  ///
+  /// See also:
+  /// https//github.com/Microsoft/vscode/blob/master/src/vs/editor/contrib/snippet/common/snippet.md
+  Snippet = 2,
+};
+
+struct CompletionItem {
+  /// The label of this completion item. By default also the text that is
+  /// inserted when selecting this completion.
+  std::string label;
+
+  /// The kind of this completion item. Based of the kind an icon is chosen by
+  /// the editor.
+  CompletionItemKind kind = CompletionItemKind::Missing;
+
+  /// A human-readable string with additional information about this item, like
+  /// type or symbol information.
+  std::string detail;
+
+  /// A human-readable string that represents a doc-comment.
+  std::string documentation;
+
+  /// A string that should be used when comparing this item with other items.
+  /// When `falsy` the label is used.
+  std::string sortText;
+
+  /// A string that should be used when filtering a set of completion items.
+  /// When `falsy` the label is used.
+  std::string filterText;
+
+  /// A string that should be inserted to a document when selecting this
+  /// completion. When `falsy` the label is used.
+  std::string insertText;
+
+  /// The format of the insert text. The format applies to both the `insertText`
+  /// property and the `newText` property of a provided `textEdit`.
+  InsertTextFormat insertTextFormat = InsertTextFormat::Missing;
+
+  /// An edit which is applied to a document when selecting this completion.
+  /// When an edit is provided `insertText` is ignored.
+  ///
+  /// Note: The range of the edit must be a single line range and it must
+  /// contain the position at which completion has been requested.
+  llvm::Optional<TextEdit> textEdit;
+
+  /// An optional array of additional text edits that are applied when selecting
+  /// this completion. Edits must not overlap with the main edit nor with
+  /// themselves.
+  std::vector<TextEdit> additionalTextEdits;
+  
+  // TODO(krasimir): The following optional fields defined by the language
+  // server protocol are unsupported:
+  //
+  // command?: Command - An optional command that is executed *after* inserting
+  //                     this completion.
+  //
+  // data?: any - A data entry field that is preserved on a completion item
+  //              between a completion and a completion resolve request.
+  static std::string unparse(const CompletionItem &P);
+};
+
 } // namespace clangd
 } // namespace clang
 

clang-tools-extra/clangd/ProtocolHandlers.cpp

@@ -178,3 +178,25 @@ void CodeActionHandler::handleMethod(llvm::yaml::MappingNode *Params,
       R"(, "result": [)" + Commands +
       R"(]})");
 }
+
+void CompletionHandler::handleMethod(llvm::yaml::MappingNode *Params,
+                                     StringRef ID) {
+  auto TDPP = TextDocumentPositionParams::parse(Params);
+  if (!TDPP) {
+    Output.log("Failed to decode TextDocumentPositionParams!\n");
+    return;
+  }
+
+  auto Items = AST.codeComplete(TDPP->textDocument.uri, TDPP->position.line,
+                                TDPP->position.character);
+  std::string Completions;
+  for (const auto &Item : Items) {
+    Completions += CompletionItem::unparse(Item);
+    Completions += ",";
+  }
+  if (!Completions.empty())
+    Completions.pop_back();
+  writeMessage(
+      R"({"jsonrpc":"2.0","id":)" + ID.str() +
+      R"(,"result":[)" + Completions + R"(]})");
+}

clang-tools-extra/clangd/ProtocolHandlers.h

@@ -36,7 +36,8 @@ struct InitializeHandler : Handler {
           "documentFormattingProvider": true,
           "documentRangeFormattingProvider": true,
           "documentOnTypeFormattingProvider": {"firstTriggerCharacter":"}","moreTriggerCharacter":[]},
-          "codeActionProvider": true
+          "codeActionProvider": true,
+          "completionProvider": {"resolveProvider": false, "triggerCharacters": [".",">"]}
         }}})");
   }
 };
@@ -114,6 +115,16 @@ private:
   ASTManager *
 };
 
+struct CompletionHandler : Handler {
+  CompletionHandler(JSONOutput &Output, ASTManager &AST)
+      : Handler(Output), AST(AST) {}
+
+  void handleMethod(llvm::yaml::MappingNode *Params, StringRef ID) override;
+
+ private:
+  ASTManager *
+};
+
 } // namespace clangd
 } // namespace clang
 

clang-tools-extra/test/clangd/completion.test

@@ -0,0 +1,69 @@
+# RUN: clangd -run-synchronously < %s | FileCheck %s
+# It is absolutely vital that this file has CRLF line endings.
+#
+Content-Length: 125
+
+{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"clangd","capabilities":{},"trace":"off"}}
+
+Content-Length: 208
+
+{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"file:///main.cpp","languageId":"cpp","version":1,"text":"#include <vector>\nint main() {\n  std::vector<int> v;\n  v.\n}\n"}}}
+
+Content-Length: 148
+
+{"jsonrpc":"2.0","id":1,"method":"textDocument/completion","params":{"textDocument":{"uri":"file:///main.cpp"},"position":{"line":3,"character":4}}}
+# The order of results returned by ASTUnit CodeComplete seems to be
+# nondeterministic, so we check regardless of order.
+#
+# CHECK: {"jsonrpc":"2.0","id":1,"result":[
+# CHECK-DAG: {"label":"_M_allocate"}
+# CHECK-DAG: {"label":"_M_allocate_and_copy"}
+# CHECK-DAG: {"label":"_M_assign_aux"}
+# CHECK-DAG: {"label":"_M_assign_dispatch"}
+# CHECK-DAG: {"label":"_M_check_len"}
+# CHECK-DAG: {"label":"_M_create_storage"
+# CHECK-DAG: {"label":"_M_deallocate"}
+# CHECK-DAG: {"label":"_M_erase_at_end"}
+# CHECK-DAG: {"label":"_M_fill_assign"}
+# CHECK-DAG: {"label":"_M_fill_initialize"}
+# CHECK-DAG: {"label":"_M_fill_insert"}
+# CHECK-DAG: {"label":"_M_get_Tp_allocator"}
+# CHECK-DAG: {"label":"_M_impl"}
+# CHECK-DAG: {"label":"_M_initialize_dispatch"}
+# CHECK-DAG: {"label":"_M_insert_aux"}
+# CHECK-DAG: {"label":"_M_insert_dispatch"}
+# CHECK-DAG: {"label":"_M_range_check"}
+# CHECK-DAG: {"label":"_M_range_initialize"}
+# CHECK-DAG: {"label":"_M_range_insert"}
+# CHECK-DAG: {"label":"_Vector_base"}
+# CHECK-DAG: {"label":"assign"}
+# CHECK-DAG: {"label":"at"}
+# CHECK-DAG: {"label":"back"}
+# CHECK-DAG: {"label":"begin"}
+# CHECK-DAG: {"label":"capacity"}
+# CHECK-DAG: {"label":"clear"}
+# CHECK-DAG: {"label":"data"}
+# CHECK-DAG: {"label":"empty"}
+# CHECK-DAG: {"label":"end"}
+# CHECK-DAG: {"label":"erase"}
+# CHECK-DAG: {"label":"front"}
+# CHECK-DAG: {"label":"get_allocator"}
+# CHECK-DAG: {"label":"insert"}
+# CHECK-DAG: {"label":"max_size"}
+# CHECK-DAG: {"label":"operator="}
+# CHECK-DAG: {"label":"operator[]"}
+# CHECK-DAG: {"label":"pop_back"}
+# CHECK-DAG: {"label":"push_back"}
+# CHECK-DAG: {"label":"rbegin"}
+# CHECK-DAG: {"label":"rend"}
+# CHECK-DAG: {"label":"reserve"}
+# CHECK-DAG: {"label":"resize"}
+# CHECK-DAG: {"label":"size"}
+# CHECK-DAG: {"label":"swap"}
+# CHECK-DAG: {"label":"vector"}
+# CHECK-DAG: {"label":"~_Vector_base"}
+# CHECK-DAG: {"label":"~vector"}
+# CHECK: ]}
+Content-Length: 44
+
+{"jsonrpc":"2.0","id":3,"method":"shutdown"}

clang-tools-extra/test/clangd/formatting.test

@@ -4,13 +4,14 @@
 Content-Length: 125
 
 {"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"clangd","capabilities":{},"trace":"off"}}
-# CHECK: Content-Length: 332
+# CHECK: Content-Length: 424
 # CHECK: {"jsonrpc":"2.0","id":0,"result":{"capabilities":{
 # CHECK:   "textDocumentSync": 1,
 # CHECK:   "documentFormattingProvider": true,
 # CHECK:   "documentRangeFormattingProvider": true,
 # CHECK:   "documentOnTypeFormattingProvider": {"firstTriggerCharacter":"}","moreTriggerCharacter":[]},
-# CHECK:   "codeActionProvider": true
+# CHECK:   "codeActionProvider": true,
+# CHECK:   "completionProvider": {"resolveProvider": false, "triggerCharacters": [".",">"]}
 # CHECK: }}}
 #
 Content-Length: 193