[clangd] Pull installed gRPC and introduce clangd-remote-(server|client)

Summary:
This patch allows using installed gRPC to build two simple tools which
currently provide the functionality of looking up the symbol by name.
remote-index-client is a simplified version of dexp which connects to
remote-index-server passes lookup requests.

I also significantly reduced the scope of this patch to prevent large changelist
and more bugs. The next steps would be:

* Extending Protocol for deep copies of Symbol and inherit RemoteIndex from
  Index to unify the interfaces
* Make remote-index-server more generic and merge the remote index client with
  dexp
* Modify Clangd to allow using remote index instead of the local one for all
  global index requests

Reviewers: sammccall

Reviewed By: sammccall

Subscribers: mgorny, ilya-biryukov, MaskRay, jkorous, arphaman, kadircet, usaxena95, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D77794

clang-tools-extra/clangd/CMakeLists.txt

@@ -153,3 +153,12 @@ if(CLANG_INCLUDE_TESTS)
 add_subdirectory(test)
 add_subdirectory(unittests)
 endif()
+
+# FIXME(kirillbobyrev): Document this in the LLVM docs once remote index is stable.
+option(CLANGD_ENABLE_REMOTE "Use gRPC library to enable remote index support for Clangd" OFF)
+set(GRPC_INSTALL_PATH "" CACHE PATH "Path to gRPC library manual installation.")
+
+if (CLANGD_ENABLE_REMOTE)
+  include(FindGRPC)
+  add_subdirectory(index/remote)
+endif()

clang-tools-extra/clangd/index/remote/CMakeLists.txt

@@ -0,0 +1,7 @@
+generate_grpc_protos(RemoteIndexProtos "Index.proto")
+
+include_directories("${CMAKE_CURRENT_BINARY_DIR}")
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../)
+
+add_subdirectory(client)
+add_subdirectory(server)

clang-tools-extra/clangd/index/remote/Index.proto

@@ -0,0 +1,19 @@
+//===--- Index.proto - Remote index Protocol Buffers definition -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+syntax = "proto3";
+
+package clang.clangd.remote;
+
+service Index {
+  rpc Lookup(LookupRequest) returns (stream LookupReply) {}
+}
+
+message LookupRequest { string id = 1; }
+
+message LookupReply { string symbol_yaml = 1; }

clang-tools-extra/clangd/index/remote/README.md

@@ -0,0 +1,59 @@
+# Clangd remote index
+
+Clangd uses a global index for project-wide code completion, navigation and
+other features.  For large projects, building this can take many hours and
+keeping it loaded uses a lot of memory.
+
+To relieve that burden, we're building remote index — a global index
+served on a different machine and shared between developers. This directory
+contains code that is used as Proof of Concept for the upcoming remote index
+feature.
+
+## Building
+
+This feature uses gRPC and Protobuf libraries, so you will need to install them.
+There are two ways of doing that.
+
+However you install dependencies, to enable this feature and build remote index
+tools you will need to set this CMake flag — `-DCLANGD_ENABLE_REMOTE=On`.
+
+### System-installed libraries
+
+On Debian-like systems gRPC and Protobuf can be installed from apt:
+
+```bash
+apt install libgrpc++-dev libprotobuf-dev protobuf-compiler protobuf-compiler-grpc
+```
+
+### Building from sources
+
+Another way of installing gRPC and Protobuf is building from sources using
+CMake. The easiest way of doing that would be to choose a directory where you
+want to install so that the installation files are not copied to system root and
+you can uninstall gRPC or use different versions of the library.
+
+```bash
+# Get source code.
+$ git clone -b v1.28.1 https://github.com/grpc/grpc
+$ cd grpc
+$ git submodule update --init
+# Choose directory where you want gRPC installation to live.
+$ export GRPC_INSTALL_PATH=/where/you/want/grpc/to/be/installed
+# Build and install gRPC to ${GRPC_INSTALL_PATH}
+$ mkdir build; cd build
+$ cmake -DgRPC_INSTALL=ON -DCMAKE_INSTALL_PREFIX=${GRPC_INSTALL_PATH} -DCMAKE_BUILD_TYPE=Release ..
+$ make install
+```
+
+This [guide](https://github.com/grpc/grpc/blob/master/BUILDING.md) goes into
+more detail on how to build gRPC from sources.
+
+By default, CMake will look for system-installed libraries when building remote
+index tools so you will have to adjust LLVM's CMake invocation. The following
+flag will inform build system that you chose this option —
+`-DGRPC_INSTALL_PATH=${GRPC_INSTALL_PATH}`.
+
+## Running
+
+The remote index isn't usable with Clangd yet, but you can try the
+proof-of-concept tools in `client/` and `server/` subdirectories.

clang-tools-extra/clangd/index/remote/client/CMakeLists.txt

@@ -0,0 +1,19 @@
+set(LLVM_LINK_COMPONENTS
+  LineEditor
+  Support
+  )
+add_clang_executable(clangd-index-client
+  Client.cpp
+  )
+target_compile_definitions(clangd-index-client PRIVATE -DGOOGLE_PROTOBUF_NO_RTTI=1)
+clang_target_link_libraries(clangd-index-client
+  PRIVATE
+  clangDaemon
+  )
+target_link_libraries(clangd-index-client
+  PRIVATE
+  RemoteIndexProtos
+
+  protobuf
+  grpc++
+  )

clang-tools-extra/clangd/index/remote/client/Client.cpp

@@ -0,0 +1,91 @@
+//===--- Client.cpp - Remote Index Client -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a simple interactive tool which can be used to manually
+// evaluate symbol search quality of Clangd index.
+//
+//===----------------------------------------------------------------------===//
+
+#include "SourceCode.h"
+#include "index/Serialization.h"
+#include "index/dex/Dex.h"
+#include "llvm/ADT/ScopeExit.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/LineEditor/LineEditor.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Signals.h"
+
+#include "grpcpp/grpcpp.h"
+
+#include "Index.grpc.pb.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+llvm::cl::opt<std::string>
+    ServerAddress("server-address",
+                  llvm::cl::desc("Address of remote index server to use."),
+                  llvm::cl::init("0.0.0.0:50051"));
+
+static const std::string Overview = R"(
+This is an **experimental** interactive tool to process user-provided search
+queries over given symbol collection obtained via clangd-indexer with the help
+of remote index server. The client will connect to remote index server and pass
+it lookup queries.
+)";
+
+class RemoteIndexClient {
+public:
+  RemoteIndexClient(std::shared_ptr<grpc::Channel> Channel)
+      : Stub(remote::Index::NewStub(Channel)) {}
+
+  void lookup(llvm::StringRef ID) {
+    llvm::outs() << "Lookup of symbol with ID " << ID << '\n';
+    remote::LookupRequest Proto;
+    Proto.set_id(ID.str());
+
+    grpc::ClientContext Context;
+    remote::LookupReply Reply;
+    std::unique_ptr<grpc::ClientReader<remote::LookupReply>> Reader(
+        Stub->Lookup(&Context, Proto));
+    while (Reader->Read(&Reply)) {
+      llvm::outs() << Reply.symbol_yaml();
+    }
+    grpc::Status Status = Reader->Finish();
+    if (Status.ok()) {
+      llvm::outs() << "lookupRequest rpc succeeded.\n";
+    } else {
+      llvm::outs() << "lookupRequest rpc failed.\n";
+    }
+  }
+
+private:
+  std::unique_ptr<remote::Index::Stub> Stub;
+};
+
+} // namespace
+} // namespace clangd
+} // namespace clang
+
+int main(int argc, const char *argv[]) {
+  using namespace clang::clangd;
+
+  llvm::cl::ParseCommandLineOptions(argc, argv, Overview);
+  llvm::cl::ResetCommandLineParser(); // We reuse it for REPL commands.
+  llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
+
+  RemoteIndexClient IndexClient(
+      grpc::CreateChannel(ServerAddress, grpc::InsecureChannelCredentials()));
+
+  llvm::LineEditor LE("remote-index-client");
+  while (llvm::Optional<std::string> Request = LE.readLine())
+    IndexClient.lookup(std::move(*Request));
+}

clang-tools-extra/clangd/index/remote/server/CMakeLists.txt

@@ -0,0 +1,20 @@
+set(LLVM_LINK_COMPONENTS
+  LineEditor
+  Support
+  )
+add_clang_executable(clangd-index-server
+  Server.cpp
+  )
+target_compile_definitions(clangd-index-server PRIVATE -DGOOGLE_PROTOBUF_NO_RTTI=1)
+clang_target_link_libraries(clangd-index-server
+  PRIVATE
+  clangDaemon
+  )
+target_link_libraries(clangd-index-server
+  PRIVATE
+  RemoteIndexProtos
+
+  protobuf
+  grpc++
+  clangDaemon
+  )

clang-tools-extra/clangd/index/remote/server/Server.cpp

@@ -0,0 +1,102 @@
+//===--- Server.cpp - gRPC-based Remote Index Server  ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "index/Index.h"
+#include "index/Serialization.h"
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/LineEditor/LineEditor.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Signals.h"
+
+#include "grpcpp/grpcpp.h"
+#include "grpcpp/health_check_service_interface.h"
+
+#include "Index.grpc.pb.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+static const std::string Overview = R"(
+This is an experimental remote index implementation. The server opens Dex and
+awaits gRPC lookup requests from the client.
+)";
+
+llvm::cl::opt<std::string> IndexPath(llvm::cl::desc("<INDEX FILE>"),
+                                     llvm::cl::Positional, llvm::cl::Required);
+
+llvm::cl::opt<std::string> ServerAddress("server-address",
+                                         llvm::cl::init("0.0.0.0:50051"));
+
+std::unique_ptr<SymbolIndex> openIndex(llvm::StringRef Index) {
+  return loadIndex(Index, /*UseIndex=*/true);
+}
+
+class RemoteIndexServer final : public remote::Index::Service {
+public:
+  RemoteIndexServer(std::unique_ptr<SymbolIndex> Index)
+      : Index(std::move(Index)) {}
+
+private:
+  grpc::Status Lookup(grpc::ServerContext *Context,
+                      const remote::LookupRequest *Request,
+                      grpc::ServerWriter<remote::LookupReply> *Reply) override {
+    llvm::outs() << "Lookup of symbol with ID " << Request->id() << '\n';
+    LookupRequest Req;
+    auto SID = SymbolID::fromStr(Request->id());
+    if (!SID) {
+      llvm::outs() << llvm::toString(SID.takeError()) << "\n";
+      return grpc::Status::CANCELLED;
+    }
+    Req.IDs.insert(*SID);
+    Index->lookup(Req, [&](const Symbol &Sym) {
+      remote::LookupReply NextSymbol;
+      NextSymbol.set_symbol_yaml(toYAML(Sym));
+      Reply->Write(NextSymbol);
+    });
+    return grpc::Status::OK;
+  }
+
+  std::unique_ptr<SymbolIndex> Index;
+};
+
+void runServer(std::unique_ptr<SymbolIndex> Index,
+               const std::string &ServerAddress) {
+  RemoteIndexServer Service(std::move(Index));
+
+  grpc::EnableDefaultHealthCheckService(true);
+  grpc::ServerBuilder Builder;
+  Builder.AddListeningPort(ServerAddress, grpc::InsecureServerCredentials());
+  Builder.RegisterService(&Service);
+  std::unique_ptr<grpc::Server> Server(Builder.BuildAndStart());
+  llvm::outs() << "Server listening on " << ServerAddress << '\n';
+
+  Server->Wait();
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
+
+int main(int argc, char *argv[]) {
+  using namespace clang::clangd;
+  llvm::cl::ParseCommandLineOptions(argc, argv, clang::clangd::Overview);
+  llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
+
+  std::unique_ptr<SymbolIndex> Index = openIndex(IndexPath);
+
+  if (!Index) {
+    llvm::outs() << "Failed to open the index.\n";
+    return -1;
+  }
+
+  runServer(std::move(Index), ServerAddress);
+}

llvm/cmake/modules/FindGRPC.cmake

@@ -0,0 +1,50 @@
+# This setup requires gRPC to be built from sources using CMake and installed to
+# ${GRPC_INSTALL_PATH} via -DCMAKE_INSTALL_PREFIX=${GRPC_INSTALL_PATH}.
+if (GRPC_INSTALL_PATH)
+  set(protobuf_MODULE_COMPATIBLE TRUE)
+  find_package(Protobuf CONFIG REQUIRED HINTS ${GRPC_INSTALL_PATH})
+  message(STATUS "Using protobuf ${protobuf_VERSION}")
+  find_package(gRPC CONFIG REQUIRED HINTS ${GRPC_INSTALL_PATH})
+  message(STATUS "Using gRPC ${gRPC_VERSION}")
+
+  include_directories(${Protobuf_INCLUDE_DIRS})
+
+  # gRPC CMake CONFIG gives the libraries slightly odd names, make them match
+  # the conventional system-installed names.
+  set_target_properties(protobuf::libprotobuf PROPERTIES IMPORTED_GLOBAL TRUE)
+  add_library(protobuf ALIAS protobuf::libprotobuf)
+  set_target_properties(gRPC::grpc++ PROPERTIES IMPORTED_GLOBAL TRUE)
+  add_library(grpc++ ALIAS gRPC::grpc++)
+
+  set(GRPC_CPP_PLUGIN $<TARGET_FILE:gRPC::grpc_cpp_plugin>)
+  set(PROTOC ${Protobuf_PROTOC_EXECUTABLE})
+else()
+  find_program(GRPC_CPP_PLUGIN grpc_cpp_plugin)
+  find_program(PROTOC protoc)
+endif()
+
+# Proto headers are generated in ${CMAKE_CURRENT_BINARY_DIR}.
+# Libraries that use these headers should adjust the include path.
+# FIXME(kirillbobyrev): Allow optional generation of gRPC code and give callers
+# control over it via additional parameters.
+function(generate_grpc_protos LibraryName ProtoFile)
+  get_filename_component(ProtoSourceAbsolutePath "${CMAKE_CURRENT_SOURCE_DIR}/${ProtoFile}" ABSOLUTE)
+  get_filename_component(ProtoSourcePath ${ProtoSourceAbsolutePath} PATH)
+
+  set(GeneratedProtoSource "${CMAKE_CURRENT_BINARY_DIR}/Index.pb.cc")
+  set(GeneratedProtoHeader "${CMAKE_CURRENT_BINARY_DIR}/Index.pb.h")
+  set(GeneratedGRPCSource "${CMAKE_CURRENT_BINARY_DIR}/Index.grpc.pb.cc")
+  set(GeneratedGRPCHeader "${CMAKE_CURRENT_BINARY_DIR}/Index.grpc.pb.h")
+  add_custom_command(
+        OUTPUT "${GeneratedProtoSource}" "${GeneratedProtoHeader}" "${GeneratedGRPCSource}" "${GeneratedGRPCHeader}"
+        COMMAND ${PROTOC}
+        ARGS --grpc_out="${CMAKE_CURRENT_BINARY_DIR}"
+          --cpp_out="${CMAKE_CURRENT_BINARY_DIR}"
+          --proto_path="${ProtoSourcePath}"
+          --plugin=protoc-gen-grpc="${GRPC_CPP_PLUGIN}"
+          "${ProtoSourceAbsolutePath}"
+          DEPENDS "${ProtoSourceAbsolutePath}")
+
+  add_library(${LibraryName} ${GeneratedProtoSource} ${GeneratedGRPCSource})
+  target_link_libraries(${LibraryName} grpc++ protobuf)
+endfunction()