Merge from 'master' to 'sycl-web' (#1)

Author: AlexeySachkov

clang-tools-extra/clangd/CMakeLists.txt

@@ -153,3 +153,12 @@ if(CLANG_INCLUDE_TESTS)
 add_subdirectory(test)
 add_subdirectory(unittests)
 endif()
+
+# FIXME(kirillbobyrev): Document this in the LLVM docs once remote index is stable.
+option(CLANGD_ENABLE_REMOTE "Use gRPC library to enable remote index support for Clangd" OFF)
+set(GRPC_INSTALL_PATH "" CACHE PATH "Path to gRPC library manual installation.")
+
+if (CLANGD_ENABLE_REMOTE)
+  include(FindGRPC)
+  add_subdirectory(index/remote)
+endif()

clang-tools-extra/clangd/CodeComplete.cpp

@@ -1836,7 +1836,7 @@ CompletionItem CodeCompletion::render(const CodeCompleteOptions &Opts) const {
   // is mainly to help LSP clients again, so that changes do not effect each
   // other.
   for (const auto &FixIt : FixIts) {
-    if (isRangeConsecutive(FixIt.range, LSP.textEdit->range)) {
+    if (FixIt.range.end == LSP.textEdit->range.start) {
       LSP.textEdit->newText = FixIt.newText + LSP.textEdit->newText;
       LSP.textEdit->range.start = FixIt.range.start;
     } else {

clang-tools-extra/clangd/SourceCode.cpp

@@ -240,26 +240,6 @@ bool isValidFileRange(const SourceManager &Mgr, SourceRange R) {
   return BeginFID.isValid() && BeginFID == EndFID && BeginOffset <= EndOffset;
 }
 
-bool halfOpenRangeContains(const SourceManager &Mgr, SourceRange R,
-                           SourceLocation L) {
-  assert(isValidFileRange(Mgr, R));
-
-  FileID BeginFID;
-  size_t BeginOffset = 0;
-  std::tie(BeginFID, BeginOffset) = Mgr.getDecomposedLoc(R.getBegin());
-  size_t EndOffset = Mgr.getFileOffset(R.getEnd());
-
-  FileID LFid;
-  size_t LOffset;
-  std::tie(LFid, LOffset) = Mgr.getDecomposedLoc(L);
-  return BeginFID == LFid && BeginOffset <= LOffset && LOffset < EndOffset;
-}
-
-bool halfOpenRangeTouches(const SourceManager &Mgr, SourceRange R,
-                          SourceLocation L) {
-  return L == R.getEnd() || halfOpenRangeContains(Mgr, R, L);
-}
-
 SourceLocation includeHashLoc(FileID IncludedFile, const SourceManager &SM) {
   assert(SM.getLocForEndOfFile(IncludedFile).isFileID());
   FileID IncludingFile;
@@ -558,11 +538,6 @@ TextEdit toTextEdit(const FixItHint &FixIt, const SourceManager &M,
   return Result;
 }
 
-bool isRangeConsecutive(const Range &Left, const Range &Right) {
-  return Left.end.line == Right.start.line &&
-         Left.end.character == Right.start.character;
-}
-
 FileDigest digest(llvm::StringRef Content) {
   uint64_t Hash{llvm::xxHash64(Content)};
   FileDigest Result;

clang-tools-extra/clangd/SourceCode.h

@@ -120,17 +120,6 @@ llvm::Optional<SourceRange> toHalfOpenFileRange(const SourceManager &Mgr,
 /// FIXME: introduce a type for source range with this invariant.
 bool isValidFileRange(const SourceManager &Mgr, SourceRange R);
 
-/// Returns true iff \p L is contained in \p R.
-/// EXPECTS: isValidFileRange(R) == true, L is a file location.
-bool halfOpenRangeContains(const SourceManager &Mgr, SourceRange R,
-                           SourceLocation L);
-
-/// Returns true iff \p L is contained in \p R or \p L is equal to the end point
-/// of \p R.
-/// EXPECTS: isValidFileRange(R) == true, L is a file location.
-bool halfOpenRangeTouches(const SourceManager &Mgr, SourceRange R,
-                          SourceLocation L);
-
 /// Returns the source code covered by the source range.
 /// EXPECTS: isValidFileRange(R) == true.
 llvm::StringRef toSourceCode(const SourceManager &SM, SourceRange R);
@@ -171,8 +160,6 @@ TextEdit toTextEdit(const FixItHint &FixIt, const SourceManager &M,
 llvm::Optional<std::string> getCanonicalPath(const FileEntry *F,
                                              const SourceManager &SourceMgr);
 
-bool isRangeConsecutive(const Range &Left, const Range &Right);
-
 /// Choose the clang-format style we should apply to a certain file.
 /// This will usually use FS to look for .clang-format directories.
 /// FIXME: should we be caching the .clang-format file search?

clang-tools-extra/clangd/index/FileIndex.cpp

@@ -174,9 +174,9 @@ FileShardedIndex::FileShardedIndex(IndexFileIn Input, PathRef HintPath)
   // not have been indexed, see SymbolCollector::processRelations for details.
   if (Index.Relations) {
     for (const auto &R : *Index.Relations) {
-      auto *File = SymbolIDToFile.lookup(R.Subject);
-      assert(File && "unknown subject in relation");
-      File->Relations.insert(&R);
+      // FIXME: RelationSlab shouldn't contain dangling relations.
+      if (auto *File = SymbolIDToFile.lookup(R.Subject))
+        File->Relations.insert(&R);
     }
   }
   // Store only the direct includes of a file in a shard.

clang-tools-extra/clangd/index/remote/CMakeLists.txt

@@ -0,0 +1,7 @@
+generate_grpc_protos(RemoteIndexProtos "Index.proto")
+
+include_directories("${CMAKE_CURRENT_BINARY_DIR}")
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../)
+
+add_subdirectory(client)
+add_subdirectory(server)

clang-tools-extra/clangd/index/remote/Index.proto

@@ -0,0 +1,19 @@
+//===--- Index.proto - Remote index Protocol Buffers definition -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+syntax = "proto3";
+
+package clang.clangd.remote;
+
+service Index {
+  rpc Lookup(LookupRequest) returns (stream LookupReply) {}
+}
+
+message LookupRequest { string id = 1; }
+
+message LookupReply { string symbol_yaml = 1; }

clang-tools-extra/clangd/index/remote/README.md

@@ -0,0 +1,59 @@
+# Clangd remote index
+
+Clangd uses a global index for project-wide code completion, navigation and
+other features.  For large projects, building this can take many hours and
+keeping it loaded uses a lot of memory.
+
+To relieve that burden, we're building remote index — a global index
+served on a different machine and shared between developers. This directory
+contains code that is used as Proof of Concept for the upcoming remote index
+feature.
+
+## Building
+
+This feature uses gRPC and Protobuf libraries, so you will need to install them.
+There are two ways of doing that.
+
+However you install dependencies, to enable this feature and build remote index
+tools you will need to set this CMake flag — `-DCLANGD_ENABLE_REMOTE=On`.
+
+### System-installed libraries
+
+On Debian-like systems gRPC and Protobuf can be installed from apt:
+
+```bash
+apt install libgrpc++-dev libprotobuf-dev protobuf-compiler protobuf-compiler-grpc
+```
+
+### Building from sources
+
+Another way of installing gRPC and Protobuf is building from sources using
+CMake. The easiest way of doing that would be to choose a directory where you
+want to install so that the installation files are not copied to system root and
+you can uninstall gRPC or use different versions of the library.
+
+```bash
+# Get source code.
+$ git clone -b v1.28.1 https://github.com/grpc/grpc
+$ cd grpc
+$ git submodule update --init
+# Choose directory where you want gRPC installation to live.
+$ export GRPC_INSTALL_PATH=/where/you/want/grpc/to/be/installed
+# Build and install gRPC to ${GRPC_INSTALL_PATH}
+$ mkdir build; cd build
+$ cmake -DgRPC_INSTALL=ON -DCMAKE_INSTALL_PREFIX=${GRPC_INSTALL_PATH} -DCMAKE_BUILD_TYPE=Release ..
+$ make install
+```
+
+This [guide](https://github.com/grpc/grpc/blob/master/BUILDING.md) goes into
+more detail on how to build gRPC from sources.
+
+By default, CMake will look for system-installed libraries when building remote
+index tools so you will have to adjust LLVM's CMake invocation. The following
+flag will inform build system that you chose this option —
+`-DGRPC_INSTALL_PATH=${GRPC_INSTALL_PATH}`.
+
+## Running
+
+The remote index isn't usable with Clangd yet, but you can try the
+proof-of-concept tools in `client/` and `server/` subdirectories.

clang-tools-extra/clangd/index/remote/client/CMakeLists.txt

@@ -0,0 +1,19 @@
+set(LLVM_LINK_COMPONENTS
+  LineEditor
+  Support
+  )
+add_clang_executable(clangd-index-client
+  Client.cpp
+  )
+target_compile_definitions(clangd-index-client PRIVATE -DGOOGLE_PROTOBUF_NO_RTTI=1)
+clang_target_link_libraries(clangd-index-client
+  PRIVATE
+  clangDaemon
+  )
+target_link_libraries(clangd-index-client
+  PRIVATE
+  RemoteIndexProtos
+
+  protobuf
+  grpc++
+  )

clang-tools-extra/clangd/index/remote/client/Client.cpp

@@ -0,0 +1,91 @@
+//===--- Client.cpp - Remote Index Client -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a simple interactive tool which can be used to manually
+// evaluate symbol search quality of Clangd index.
+//
+//===----------------------------------------------------------------------===//
+
+#include "SourceCode.h"
+#include "index/Serialization.h"
+#include "index/dex/Dex.h"
+#include "llvm/ADT/ScopeExit.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/LineEditor/LineEditor.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Signals.h"
+
+#include "grpcpp/grpcpp.h"
+
+#include "Index.grpc.pb.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+llvm::cl::opt<std::string>
+    ServerAddress("server-address",
+                  llvm::cl::desc("Address of remote index server to use."),
+                  llvm::cl::init("0.0.0.0:50051"));
+
+static const std::string Overview = R"(
+This is an **experimental** interactive tool to process user-provided search
+queries over given symbol collection obtained via clangd-indexer with the help
+of remote index server. The client will connect to remote index server and pass
+it lookup queries.
+)";
+
+class RemoteIndexClient {
+public:
+  RemoteIndexClient(std::shared_ptr<grpc::Channel> Channel)
+      : Stub(remote::Index::NewStub(Channel)) {}
+
+  void lookup(llvm::StringRef ID) {
+    llvm::outs() << "Lookup of symbol with ID " << ID << '\n';
+    remote::LookupRequest Proto;
+    Proto.set_id(ID.str());
+
+    grpc::ClientContext Context;
+    remote::LookupReply Reply;
+    std::unique_ptr<grpc::ClientReader<remote::LookupReply>> Reader(
+        Stub->Lookup(&Context, Proto));
+    while (Reader->Read(&Reply)) {
+      llvm::outs() << Reply.symbol_yaml();
+    }
+    grpc::Status Status = Reader->Finish();
+    if (Status.ok()) {
+      llvm::outs() << "lookupRequest rpc succeeded.\n";
+    } else {
+      llvm::outs() << "lookupRequest rpc failed.\n";
+    }
+  }
+
+private:
+  std::unique_ptr<remote::Index::Stub> Stub;
+};
+
+} // namespace
+} // namespace clangd
+} // namespace clang
+
+int main(int argc, const char *argv[]) {
+  using namespace clang::clangd;
+
+  llvm::cl::ParseCommandLineOptions(argc, argv, Overview);
+  llvm::cl::ResetCommandLineParser(); // We reuse it for REPL commands.
+  llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
+
+  RemoteIndexClient IndexClient(
+      grpc::CreateChannel(ServerAddress, grpc::InsecureChannelCredentials()));
+
+  llvm::LineEditor LE("remote-index-client");
+  while (llvm::Optional<std::string> Request = LE.readLine())
+    IndexClient.lookup(std::move(*Request));
+}

clang-tools-extra/clangd/index/remote/server/CMakeLists.txt

@@ -0,0 +1,20 @@
+set(LLVM_LINK_COMPONENTS
+  LineEditor
+  Support
+  )
+add_clang_executable(clangd-index-server
+  Server.cpp
+  )
+target_compile_definitions(clangd-index-server PRIVATE -DGOOGLE_PROTOBUF_NO_RTTI=1)
+clang_target_link_libraries(clangd-index-server
+  PRIVATE
+  clangDaemon
+  )
+target_link_libraries(clangd-index-server
+  PRIVATE
+  RemoteIndexProtos
+
+  protobuf
+  grpc++
+  clangDaemon
+  )