[clang][MBD] set up module build daemon infrastructure

clang/include/clang/Basic/DiagnosticFrontendKinds.td

@@ -267,6 +267,27 @@ def err_test_module_file_extension_version : Error<
   "test module file extension '%0' has different version (%1.%2) than expected "
   "(%3.%4)">;
 
+// Module Build Daemon
+def err_path_length :
+  Error<"path '%0' is longer then the max length of %1">,
+  DefaultFatal;
+def err_mbd_handshake :
+  Error<"attempt to handshake with module build daemon has failed: %0">,
+  DefaultFatal;
+def err_mbd_connect :
+  Error<"attempt to connect to module build daemon has failed: %0">,
+  DefaultFatal;
+def remark_mbd_spawn :
+  Remark<"successfully spawned module build daemon">,
+  InGroup<ModuleBuildDaemon>;
+def remark_mbd_connection :
+  Remark<"successfully connected to module build daemon at %0">,
+  InGroup<ModuleBuildDaemon>;
+def remark_mbd_handshake :
+  Remark<"clang invocation responsible for %0 successfully completed handshake "
+         "with module build daemon">,
+  InGroup<ModuleBuildDaemon>;
+
 def warn_eagerly_load_for_standard_cplusplus_modules : Warning<
   "the form '-fmodule-file=<BMI-path>' is deprecated for standard C++ named modules;"
   "consider to use '-fmodule-file=<module-name>=<BMI-path>' instead">,

clang/include/clang/Basic/DiagnosticGroups.td

@@ -552,6 +552,7 @@ def MissingFieldInitializers : DiagGroup<"missing-field-initializers",
                                          [MissingDesignatedFieldInitializers]>;
 def ModuleLock : DiagGroup<"module-lock">;
 def ModuleBuild : DiagGroup<"module-build">;
+def ModuleBuildDaemon : DiagGroup<"module-build-daemon">;
 def ModuleImport : DiagGroup<"module-import">;
 def ModuleConflict : DiagGroup<"module-conflict">;
 def ModuleFileExtension : DiagGroup<"module-file-extension">;

clang/include/clang/Driver/Options.td

@@ -3110,6 +3110,19 @@ defm declspec : BoolOption<"f", "declspec",
   NegFlag<SetFalse, [], [ClangOption], "Disallow">,
   BothFlags<[], [ClangOption, CC1Option],
           " __declspec as a keyword">>, Group<f_clang_Group>;
+
+def fmodule_build_daemon : Flag<["-"], "fmodule-build-daemon">, Group<f_Group>,
+  Flags<[NoXarchOption]>,
+  Visibility<[ClangOption, CC1Option]>,
+  HelpText<"Enables module build daemon functionality">,
+  MarshallingInfoFlag<FrontendOpts<"ModuleBuildDaemon">>;
+def fmodule_build_daemon_EQ : Joined<["-"], "fmodule-build-daemon=">, Group<f_Group>,
+  Flags<[NoXarchOption]>,
+  Visibility<[ClangOption, CC1Option]>,
+  HelpText<"Enables module build daemon functionality and defines the path to "
+           "where the module build daemon will write log and socket files">,
+  MarshallingInfoString<FrontendOpts<"ModuleBuildDaemonPath">>;
+
 def fmodules_cache_path : Joined<["-"], "fmodules-cache-path=">, Group<i_Group>,
   Flags<[]>, Visibility<[ClangOption, CC1Option]>,
   MetaVarName<"<directory>">,

clang/include/clang/Frontend/FrontendOptions.h

@@ -411,6 +411,10 @@ class FrontendOptions {
   LLVM_PREFERRED_TYPE(bool)
   unsigned EmitPrettySymbolGraphs : 1;
 
+  /// Connect to module build daemon.
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned ModuleBuildDaemon : 1;
+
   /// Whether to generate reduced BMI for C++20 named modules.
   LLVM_PREFERRED_TYPE(bool)
   unsigned GenReducedBMI : 1;
@@ -504,6 +508,10 @@ class FrontendOptions {
   /// The output file, if any.
   std::string OutputFile;
 
+  /// If given, the path to the module build daemon's output files and socket
+  /// address
+  std::string ModuleBuildDaemonPath;
+
   /// If given, the new suffix for fix-it rewritten files.
   std::string FixItSuffix;
 

clang/include/clang/Tooling/ModuleBuildDaemon/Frontend.h

@@ -0,0 +1,36 @@
+//===----------------------------- Frontend.h -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_FRONTEND_H
+#define LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_FRONTEND_H
+
+#include "clang/Frontend/CompilerInvocation.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/raw_socket_stream.h"
+
+namespace clang::tooling::cc1modbuildd {
+
+llvm::Error attemptHandshake(llvm::raw_socket_stream &Client,
+                             clang::DiagnosticsEngine &Diag);
+
+llvm::Error spawnModuleBuildDaemon(const clang::CompilerInvocation &Clang,
+                                   const char *Argv0,
+                                   clang::DiagnosticsEngine &Diag,
+                                   std::string BasePath);
+
+llvm::Expected<std::unique_ptr<llvm::raw_socket_stream>>
+getModuleBuildDaemon(const clang::CompilerInvocation &Clang, const char *Argv0,
+                     clang::DiagnosticsEngine &Diag, llvm::StringRef BasePath);
+
+void spawnModuleBuildDaemonAndHandshake(const clang::CompilerInvocation &Clang,
+                                        const char *Argv0,
+                                        clang::DiagnosticsEngine &Diag);
+
+} // namespace clang::tooling::cc1modbuildd
+
+#endif // LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_CLIENT_H

clang/include/clang/Tooling/ModuleBuildDaemon/SocketSupport.h

@@ -0,0 +1,131 @@
+//===------------------------- SocketMsgSupport.h -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_SOCKETMSGSUPPORT_H
+#define LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_SOCKETMSGSUPPORT_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/YAMLParser.h"
+#include "llvm/Support/YAMLTraits.h"
+#include "llvm/Support/raw_socket_stream.h"
+
+namespace clang::tooling::cc1modbuildd {
+
+enum class ActionType { HANDSHAKE };
+enum class StatusType { REQUEST, SUCCESS, FAILURE };
+
+struct BaseMsg {
+  ActionType MsgAction;
+  StatusType MsgStatus;
+
+  BaseMsg() = default;
+  BaseMsg(ActionType Action, StatusType Status)
+      : MsgAction(Action), MsgStatus(Status) {}
+};
+
+struct HandshakeMsg : public BaseMsg {
+  HandshakeMsg() = default;
+  HandshakeMsg(ActionType Action, StatusType Status)
+      : BaseMsg(Action, Status) {}
+};
+
+llvm::Expected<std::string>
+readBufferFromSocket(llvm::raw_socket_stream &Socket);
+llvm::Error writeBufferToSocket(llvm::raw_socket_stream &Socket,
+                                llvm::StringRef Buffer);
+
+template <typename T> std::string convertMsgStructToBuffer(T MsgStruct) {
+  static_assert(std::is_base_of<cc1modbuildd::BaseMsg, T>::value);
+
+  std::string Buffer;
+  llvm::raw_string_ostream OS(Buffer);
+  llvm::yaml::Output YamlOut(OS);
+
+  // TODO confirm yaml::Output does not have any error messages
+  YamlOut << MsgStruct;
+
+  return Buffer;
+}
+
+template <typename T>
+llvm::Expected<T> convertBufferToMsgStruct(llvm::StringRef Buffer) {
+  static_assert(std::is_base_of<cc1modbuildd::BaseMsg, T>::value);
+
+  T MsgStruct;
+  llvm::yaml::Input YamlIn(Buffer);
+  YamlIn >> MsgStruct;
+
+  // YamlIn.error() dumps an error message if there is one
+  if (YamlIn.error())
+    return llvm::make_error<llvm::StringError>(
+        "Syntax or semantic error during YAML parsing",
+        llvm::inconvertibleErrorCode());
+
+  return MsgStruct;
+}
+
+template <typename T>
+llvm::Expected<T> readMsgStructFromSocket(llvm::raw_socket_stream &Socket) {
+  static_assert(std::is_base_of<cc1modbuildd::BaseMsg, T>::value);
+
+  llvm::Expected<std::string> MaybeBuffer = readBufferFromSocket(Socket);
+  if (!MaybeBuffer)
+    return std::move(MaybeBuffer.takeError());
+  std::string Buffer = std::move(*MaybeBuffer);
+
+  llvm::Expected<T> MaybeMsgStruct = convertBufferToMsgStruct<T>(Buffer);
+  if (!MaybeMsgStruct)
+    return std::move(MaybeMsgStruct.takeError());
+  return std::move(*MaybeMsgStruct);
+}
+
+template <typename T>
+llvm::Error writeMsgStructToSocket(llvm::raw_socket_stream &Socket,
+                                   T MsgStruct) {
+  static_assert(std::is_base_of<cc1modbuildd::BaseMsg, T>::value);
+
+  std::string Buffer = convertMsgStructToBuffer(MsgStruct);
+  if (llvm::Error Err = writeBufferToSocket(Socket, Buffer))
+    return Err;
+  return llvm::Error::success();
+}
+
+} // namespace clang::tooling::cc1modbuildd
+
+namespace llvm {
+namespace yaml {
+template <>
+struct ScalarEnumerationTraits<clang::tooling::cc1modbuildd::StatusType> {
+  static void enumeration(IO &Io, clang::tooling::cc1modbuildd::StatusType &S) {
+    Io.enumCase(S, "REQUEST",
+                clang::tooling::cc1modbuildd::StatusType::REQUEST);
+    Io.enumCase(S, "SUCCESS",
+                clang::tooling::cc1modbuildd::StatusType::SUCCESS);
+    Io.enumCase(S, "FAILURE",
+                clang::tooling::cc1modbuildd::StatusType::FAILURE);
+  }
+};
+
+template <>
+struct ScalarEnumerationTraits<clang::tooling::cc1modbuildd::ActionType> {
+  static void enumeration(IO &Io, clang::tooling::cc1modbuildd::ActionType &A) {
+    Io.enumCase(A, "HANDSHAKE",
+                clang::tooling::cc1modbuildd::ActionType::HANDSHAKE);
+  }
+};
+
+template <> struct MappingTraits<clang::tooling::cc1modbuildd::HandshakeMsg> {
+  static void mapping(IO &Io, clang::tooling::cc1modbuildd::HandshakeMsg &H) {
+    Io.mapRequired("Action", H.MsgAction);
+    Io.mapRequired("Status", H.MsgStatus);
+  }
+};
+} // namespace yaml
+} // namespace llvm
+#endif // LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_SOCKETMSGSUPPORT_H

clang/include/clang/Tooling/ModuleBuildDaemon/Utils.h

@@ -0,0 +1,56 @@
+//===------------------------------ Utils.h -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Functions required by both the frontend and the module build daemon
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_UTILS_H
+#define LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_UTILS_H
+
+#include "llvm/Support/Error.h"
+
+#include <chrono>
+#include <string>
+
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+// winsock2.h must be included before afunix.h
+// clang-format off
+#include <winsock2.h>
+#include <afunix.h>
+// clang-format on
+#else
+#include <sys/un.h>
+#endif
+
+namespace clang::tooling::cc1modbuildd {
+
+constexpr std::string_view SocketFileName = "mbd.sock";
+constexpr std::string_view StdoutFileName = "mbd.out";
+constexpr std::string_view StderrFileName = "mbd.err";
+constexpr std::string_view ModuleBuildDaemonFlag = "-cc1modbuildd";
+
+// A llvm::raw_socket_stream uses sockaddr_un
+constexpr size_t SocketAddrMaxLength = sizeof(sockaddr_un::sun_path);
+
+constexpr size_t BasePathMaxLength =
+    SocketAddrMaxLength - SocketFileName.length();
+
+// Get a temprary location where the daemon can store log files and a socket
+// address. Of the format /tmp/clang-<BLAKE3HashOfClangFullVersion>/
+std::string getBasePath();
+
+// Check if the user provided BasePath is short enough
+bool validBasePathLength(llvm::StringRef);
+
+} // namespace clang::tooling::cc1modbuildd
+
+#endif

clang/lib/Driver/ToolChains/Clang.cpp

@@ -3982,6 +3982,19 @@ static bool RenderModulesOptions(Compilation &C, const Driver &D,
   bool HaveStdCXXModules = IsCXX && HaveStd20;
   bool HaveModules = HaveStdCXXModules;
 
+  // -fmodule-build-daemon enables module build daemon functionality
+  if (Args.hasArg(options::OPT_fmodule_build_daemon))
+    Args.AddLastArg(CmdArgs, options::OPT_fmodule_build_daemon);
+
+  // by default module build daemon socket address and output files are saved
+  // under /tmp/ but that can be overridden by providing the
+  // -fmodule-build-daemon=<path> flag
+  if (Arg *A = Args.getLastArg(options::OPT_fmodule_build_daemon_EQ)) {
+    CmdArgs.push_back(
+        Args.MakeArgString(Twine("-fmodule-build-daemon=") + A->getValue()));
+    CmdArgs.push_back("-fmodule-build-daemon");
+  }
+
   // -fmodules enables the use of precompiled modules (off by default).
   // Users can pass -fno-cxx-modules to turn off modules support for
   // C++/Objective-C++ programs.

clang/lib/Tooling/CMakeLists.txt

@@ -12,6 +12,7 @@ add_subdirectory(ASTDiff)
 add_subdirectory(Syntax)
 add_subdirectory(DependencyScanning)
 add_subdirectory(Transformer)
+add_subdirectory(ModuleBuildDaemon)
 
 add_clang_library(clangTooling
   AllTUsExecution.cpp

clang/lib/Tooling/ModuleBuildDaemon/CMakeLists.txt

@@ -0,0 +1,9 @@
+set(LLVM_LINK_COMPONENTS
+  Support
+  )
+
+add_clang_library(clangModuleBuildDaemon
+  Frontend.cpp
+  SocketSupport.cpp
+  Utils.cpp
+  )