[flang][driver] Add the new flang compiler and frontend drivers

Summary:

This is the first patch implementing the new Flang driver as outlined in [1],
[2] & [3]. It creates Flang driver (`flang-new`) and Flang frontend driver
(`flang-new -fc1`). These will be renamed as `flang` and `flang -fc1` once the
current Flang throwaway driver, `flang`, can be replaced with `flang-new`.

Currently only 2 options are supported: `-help` and `--version`.

`flang-new` is implemented in terms of libclangDriver, defaulting the driver
mode to `FlangMode` (added to libclangDriver in [4]). This ensures that the
driver runs in Flang mode regardless of the name of the binary inferred from
argv[0].

The design of the new Flang compiler and frontend drivers is inspired by it
counterparts in Clang [3]. Currently, the new Flang compiler and frontend
drivers re-use Clang libraries: clangBasic, clangDriver and clangFrontend.

To identify Flang options, this patch adds FlangOption/FC1Option enums.
Driver::printHelp is updated so that `flang-new` prints only Flang options.
The new Flang driver is disabled by default. To enable it, set
`-DBUILD_FLANG_NEW_DRIVER=ON` when configuring CMake and add clang to
`LLVM_ENABLE_PROJECTS` (e.g. -DLLVM_ENABLE_PROJECTS=“clang;flang;mlir”).

[1] “RFC: new Flang driver - next steps”
http://lists.llvm.org/pipermail/flang-dev/2020-July/000470.html
[2] “RFC: Adding a fortran mode to the clang driver for flang”
http://lists.llvm.org/pipermail/cfe-dev/2019-June/062669.html
[3] “RFC: refactoring libclangDriver/libclangFrontend to share with Flang”
http://lists.llvm.org/pipermail/cfe-dev/2020-July/066393.html
[4] https://reviews.llvm.org/rG6bf55804924d5a1d902925ad080b1a2b57c5c75c

co-authored-by: Andrzej Warzynski <[email protected]>

Reviewed By: richard.barton.arm, sameeranjoshi

Differential Revision: https://reviews.llvm.org/D86089

Author: CarolineConcatto

clang/include/clang/Driver/Driver.h

@@ -301,7 +301,7 @@ class Driver {
                                       StringRef CustomResourceDir = "");
 
   Driver(StringRef ClangExecutable, StringRef TargetTriple,
-         DiagnosticsEngine &Diags,
+         DiagnosticsEngine &Diags, std::string Title = "clang LLVM compiler",
          IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS = nullptr);
 
   /// @name Accessors

clang/include/clang/Driver/Options.h

@@ -34,7 +34,9 @@ enum ClangFlags {
   CC1AsOption = (1 << 11),
   NoDriverOption = (1 << 12),
   LinkOption = (1 << 13),
-  Ignored = (1 << 14),
+  FlangOption = (1 << 14),
+  FC1Option = (1 << 15),
+  Ignored = (1 << 16),
 };
 
 enum ID {

clang/include/clang/Driver/Options.td

@@ -56,6 +56,13 @@ def NoDriverOption : OptionFlag;
 // be used), add this flag.
 def LinkOption : OptionFlag;
 
+// FlangOption - This is considered a "core" Flang option, available in
+// flang mode.
+def FlangOption : OptionFlag;
+
+// FC1Option - This option should be accepted by flang -fc1.
+def FC1Option : OptionFlag;
+
 // A short name to show in documentation. The name will be interpreted as rST.
 class DocName<string name> { string DocName = name; }
 
@@ -2100,7 +2107,7 @@ def gno_embed_source : Flag<["-"], "gno-embed-source">, Group<g_flags_Group>,
     Flags<[DriverOption]>,
     HelpText<"Restore the default behavior of not embedding source text in DWARF debug sections">;
 def headerpad__max__install__names : Joined<["-"], "headerpad_max_install_names">;
-def help : Flag<["-", "--"], "help">, Flags<[CC1Option,CC1AsOption]>,
+def help : Flag<["-", "--"], "help">, Flags<[CC1Option,CC1AsOption, FC1Option, FlangOption]>,
   HelpText<"Display available options">;
 def ibuiltininc : Flag<["-"], "ibuiltininc">,
   HelpText<"Enable builtin #include directories even when -nostdinc is used "
@@ -3049,7 +3056,8 @@ def _rtlib : Separate<["--"], "rtlib">, Alias<rtlib_EQ>;
 def _serialize_diags : Separate<["-", "--"], "serialize-diagnostics">, Flags<[DriverOption]>,
   HelpText<"Serialize compiler diagnostics to a file">;
 // We give --version different semantics from -version.
-def _version : Flag<["--"], "version">, Flags<[CoreOption, CC1Option]>,
+def _version : Flag<["--"], "version">,
+  Flags<[CoreOption, CC1Option, FC1Option, FlangOption]>,
   HelpText<"Print version information">;
 def _signed_char : Flag<["--"], "signed-char">, Alias<fsigned_char>;
 def _std : Separate<["--"], "std">, Alias<std_EQ>;

clang/lib/Driver/Driver.cpp

@@ -128,12 +128,12 @@ std::string Driver::GetResourcesPath(StringRef BinaryPath,
 }
 
 Driver::Driver(StringRef ClangExecutable, StringRef TargetTriple,
-               DiagnosticsEngine &Diags,
+               DiagnosticsEngine &Diags, std::string Title,
                IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS)
     : Diags(Diags), VFS(std::move(VFS)), Mode(GCCMode),
       SaveTemps(SaveTempsNone), BitcodeEmbed(EmbedNone), LTOMode(LTOK_None),
       ClangExecutable(ClangExecutable), SysRoot(DEFAULT_SYSROOT),
-      DriverTitle("clang LLVM compiler"), CCPrintOptionsFilename(nullptr),
+      DriverTitle(Title), CCPrintOptionsFilename(nullptr),
       CCPrintHeadersFilename(nullptr), CCLogDiagnosticsFilename(nullptr),
       CCCPrintBindings(false), CCPrintOptions(false), CCPrintHeaders(false),
       CCLogDiagnostics(false), CCGenDiagnostics(false),
@@ -1571,16 +1571,23 @@ void Driver::PrintHelp(bool ShowHidden) const {
   if (!ShowHidden)
     ExcludedFlagsBitmask |= HelpHidden;
 
+  if (IsFlangMode())
+    IncludedFlagsBitmask |= options::FlangOption;
+
   std::string Usage = llvm::formatv("{0} [options] file...", Name).str();
   getOpts().PrintHelp(llvm::outs(), Usage.c_str(), DriverTitle.c_str(),
                       IncludedFlagsBitmask, ExcludedFlagsBitmask,
                       /*ShowAllAliases=*/false);
 }
 
 void Driver::PrintVersion(const Compilation &C, raw_ostream &OS) const {
-  // FIXME: The following handlers should use a callback mechanism, we don't
-  // know what the client would like to do.
-  OS << getClangFullVersion() << '\n';
+  if (IsFlangMode()) {
+    OS << getClangToolFullVersion("flang-new") << '\n';
+  } else {
+    // FIXME: The following handlers should use a callback mechanism, we don't
+    // know what the client would like to do.
+    OS << getClangFullVersion() << '\n';
+  }
   const ToolChain &TC = C.getDefaultToolChain();
   OS << "Target: " << TC.getTripleString() << '\n';
 
@@ -1618,7 +1625,7 @@ void Driver::HandleAutocompletions(StringRef PassedFlags) const {
   std::vector<std::string> SuggestedCompletions;
   std::vector<std::string> Flags;
 
-  unsigned short DisableFlags =
+  unsigned int DisableFlags =
       options::NoDriverOption | options::Unsupported | options::Ignored;
 
   // Distinguish "--autocomplete=-someflag" and "--autocomplete=-someflag,"

clang/lib/Driver/ToolChains/Flang.cpp

@@ -69,11 +69,13 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back(Input.getFilename());
 
   const auto& D = C.getDriver();
-  const char* Exec = Args.MakeArgString(D.GetProgramPath("flang", TC));
+  // TODO: Replace flang-new with flang once the new driver replaces the
+  // throwaway driver
+  const char *Exec = Args.MakeArgString(D.GetProgramPath("flang-new", TC));
   C.addCommand(std::make_unique<Command>(
       JA, *this, ResponseFileSupport::AtFileUTF8(), Exec, CmdArgs, Inputs));
 }
 
-Flang::Flang(const ToolChain &TC) : Tool("flang", "flang frontend", TC) {}
+Flang::Flang(const ToolChain &TC) : Tool("flang-new", "flang frontend", TC) {}
 
 Flang::~Flang() {}

clang/lib/Frontend/CreateInvocationFromCommandLine.cpp

@@ -40,8 +40,8 @@ std::unique_ptr<CompilerInvocation> clang::createInvocationFromCommandLine(
   Args.push_back("-fsyntax-only");
 
   // FIXME: We shouldn't have to pass in the path info.
-  driver::Driver TheDriver(Args[0], llvm::sys::getDefaultTargetTriple(),
-                           *Diags, VFS);
+  driver::Driver TheDriver(Args[0], llvm::sys::getDefaultTargetTriple(), *Diags,
+                           "clang LLVM compiler", VFS);
 
   // Don't check that inputs exist, they may have been remapped.
   TheDriver.setCheckInputsExist(false);

clang/lib/Tooling/Tooling.cpp

@@ -78,7 +78,7 @@ newDriver(DiagnosticsEngine *Diagnostics, const char *BinaryName,
           IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) {
   driver::Driver *CompilerDriver =
       new driver::Driver(BinaryName, llvm::sys::getDefaultTargetTriple(),
-                         *Diagnostics, std::move(VFS));
+                         *Diagnostics, "clang LLVM compiler", std::move(VFS));
   CompilerDriver->setTitle("clang_based_tool");
   return CompilerDriver;
 }

clang/test/Driver/flang/flang.f90

@@ -13,7 +13,7 @@
 ! * (no type specified, resulting in an object file)
 
 ! All invocations should begin with flang -fc1, consume up to here.
-! ALL-LABEL: "{{[^"]*}}flang" "-fc1"
+! ALL-LABEL: "{{[^"]*}}flang-new" "-fc1"
 
 ! Check that f90 files are not treated as "previously preprocessed"
 ! ... in --driver-mode=flang.

clang/test/Driver/flang/flang_ucase.F90

@@ -13,7 +13,7 @@
 ! * (no type specified, resulting in an object file)
 
 ! All invocations should begin with flang -fc1, consume up to here.
-! ALL-LABEL: "{{[^"]*}}flang" "-fc1"
+! ALL-LABEL: "{{[^"]*}}flang-new" "-fc1"
 
 ! Check that f90 files are not treated as "previously preprocessed"
 ! ... in --driver-mode=flang.

clang/test/Driver/flang/multiple-inputs-mixed.f90

@@ -1,7 +1,7 @@
 ! Check that flang can handle mixed C and fortran inputs.
 
 ! RUN: %clang --driver-mode=flang -### -fsyntax-only %S/Inputs/one.f90 %S/Inputs/other.c 2>&1 | FileCheck --check-prefixes=CHECK-SYNTAX-ONLY %s
-! CHECK-SYNTAX-ONLY-LABEL: "{{[^"]*}}flang{{[^"/]*}}" "-fc1"
+! CHECK-SYNTAX-ONLY-LABEL: "{{[^"]*}}flang-new{{[^"/]*}}" "-fc1"
 ! CHECK-SYNTAX-ONLY: "{{[^"]*}}/Inputs/one.f90"
 ! CHECK-SYNTAX-ONLY-LABEL: "{{[^"]*}}clang{{[^"/]*}}" "-cc1"
 ! CHECK-SYNTAX-ONLY: "{{[^"]*}}/Inputs/other.c"

clang/test/Driver/flang/multiple-inputs.f90

@@ -1,7 +1,7 @@
 ! Check that flang driver can handle multiple inputs at once.
 
 ! RUN: %clang --driver-mode=flang -### -fsyntax-only %S/Inputs/one.f90 %S/Inputs/two.f90 2>&1 | FileCheck --check-prefixes=CHECK-SYNTAX-ONLY %s
-! CHECK-SYNTAX-ONLY-LABEL: "{{[^"]*}}flang" "-fc1"
+! CHECK-SYNTAX-ONLY-LABEL: "{{[^"]*}}flang-new" "-fc1"
 ! CHECK-SYNTAX-ONLY: "{{[^"]*}}/Inputs/one.f90"
-! CHECK-SYNTAX-ONLY-LABEL: "{{[^"]*}}flang" "-fc1"
+! CHECK-SYNTAX-ONLY-LABEL: "{{[^"]*}}flang-new" "-fc1"
 ! CHECK-SYNTAX-ONLY: "{{[^"]*}}/Inputs/two.f90"

clang/unittests/Driver/SanitizerArgsTest.cpp

@@ -57,7 +57,7 @@ class SanitizerArgsTest : public ::testing::Test {
         new DiagnosticIDs, Opts,
         new TextDiagnosticPrinter(llvm::errs(), Opts.get()));
     DriverInstance.emplace(ClangBinary, "x86_64-unknown-linux-gnu", Diags,
-                           prepareFS(ExtraFiles));
+                           "clang LLVM compiler", prepareFS(ExtraFiles));
 
     std::vector<const char *> Args = {ClangBinary};
     for (const auto &A : ExtraArgs)

clang/unittests/Driver/ToolChainTest.cpp

@@ -35,7 +35,7 @@ TEST(ToolChainTest, VFSGCCInstallation) {
   IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> InMemoryFileSystem(
       new llvm::vfs::InMemoryFileSystem);
   Driver TheDriver("/bin/clang", "arm-linux-gnueabihf", Diags,
-                   InMemoryFileSystem);
+                   "clang LLVM compiler", InMemoryFileSystem);
 
   const char *EmptyFiles[] = {
       "foo.cpp",
@@ -89,7 +89,7 @@ TEST(ToolChainTest, VFSGCCInstallationRelativeDir) {
   IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> InMemoryFileSystem(
       new llvm::vfs::InMemoryFileSystem);
   Driver TheDriver("/home/test/bin/clang", "arm-linux-gnueabi", Diags,
-                   InMemoryFileSystem);
+                   "clang LLVM compiler", InMemoryFileSystem);
 
   const char *EmptyFiles[] = {
       "foo.cpp", "/home/test/lib/gcc/arm-linux-gnueabi/4.6.1/crtbegin.o",
@@ -130,13 +130,13 @@ TEST(ToolChainTest, DefaultDriverMode) {
       new llvm::vfs::InMemoryFileSystem);
 
   Driver CCDriver("/home/test/bin/clang", "arm-linux-gnueabi", Diags,
-                  InMemoryFileSystem);
+                  "clang LLVM compiler", InMemoryFileSystem);
   CCDriver.setCheckInputsExist(false);
   Driver CXXDriver("/home/test/bin/clang++", "arm-linux-gnueabi", Diags,
-                   InMemoryFileSystem);
+                   "clang LLVM compiler", InMemoryFileSystem);
   CXXDriver.setCheckInputsExist(false);
   Driver CLDriver("/home/test/bin/clang-cl", "arm-linux-gnueabi", Diags,
-                  InMemoryFileSystem);
+                  "clang LLVM compiler", InMemoryFileSystem);
   CLDriver.setCheckInputsExist(false);
 
   std::unique_ptr<Compilation> CC(CCDriver.BuildCompilation(

flang/CMakeLists.txt

@@ -17,6 +17,7 @@ if (POLICY CMP0077)
 endif()
 
 option(LINK_WITH_FIR "Link driver with FIR and LLVM" ON)
+option(FLANG_BUILD_NEW_DRIVER "Build the flang compiler driver" OFF)
 
 # Flang requires C++17.
 set(CMAKE_CXX_STANDARD 17)
@@ -61,6 +62,12 @@ if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
   get_filename_component(LLVM_DIR_ABSOLUTE ${LLVM_DIR} REALPATH)
   list(APPEND CMAKE_MODULE_PATH ${LLVM_DIR_ABSOLUTE})
 
+  if(FLANG_BUILD_NEW_DRIVER)
+    # TODO: Remove when libclangDriver is lifted out of Clang
+    list(APPEND CMAKE_MODULE_PATH ${CLANG_DIR})
+    find_package(Clang REQUIRED HINTS "${CLANG_DIR}")
+  endif()
+
   # If LLVM links to zlib we need the imported targets so we can too.
   if(LLVM_ENABLE_ZLIB)
     find_package(ZLIB REQUIRED)
@@ -200,6 +207,21 @@ else()
   endif()
 endif()
 
+if(FLANG_BUILD_NEW_DRIVER)
+    # TODO: Remove when libclangDriver is lifted out of Clang
+    if(FLANG_STANDALONE_BUILD)
+      set(CLANG_INCLUDE_DIR ${CLANG_INCLUDE_DIRS} )
+      # No need to specify TableGen output dir as that's embedded in CLANG_DIR
+    else()
+      set(CLANG_INCLUDE_DIR ${LLVM_MAIN_SRC_DIR}/../clang/include )
+      # Specify TableGen output dir for things like DiagnosticCommonKinds.inc,
+      # DiagnosticDriverKinds.inc (required for reporting diagnostics)
+      set(CLANG_TABLEGEN_OUTPUT_DIR ${CMAKE_BINARY_DIR}/tools/clang/include)
+      include_directories(SYSTEM ${CLANG_TABLEGEN_OUTPUT_DIR})
+    endif()
+    include_directories(SYSTEM ${CLANG_INCLUDE_DIR})
+endif()
+
 if(LINK_WITH_FIR)
   # tco tool and FIR lib output directories
   if(FLANG_STANDALONE_BUILD)

flang/README.md

@@ -143,6 +143,21 @@ cd ~/flang/build
 cmake -DLLVM_DIR=$LLVM -DMLIR_DIR=$MLIR ~/flang/src
 make
 ```
+
+### Build The New Flang Driver
+The new Flang driver, `flang-new`, is currently under active development and
+should be considered as an experimental feature. For this reason it is disabled
+by default. This will change once the new driver replaces the _throwaway_
+driver, `flang`.
+
+In order to build the new driver, add `-DBUILD_FLANG_NEW_DRIVER=ON` to your
+CMake invocation line. Additionally, when building out-of-tree, use `CLANG_DIR`
+(similarly to `LLVM_DIR` and `MLIR_DIR`) to find the installed Clang
+components.
+
+**Note:** `CLANG_DIR` is only required when building the new Flang driver,
+which currently depends on Clang.
+
 # How to Run Tests
 
 Flang supports 2 different categories of tests

flang/include/flang/Frontend/CompilerInstance.h

@@ -0,0 +1,105 @@
+//===-- CompilerInstance.h - Flang Compiler Instance ------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_FLANG_FRONTEND_COMPILERINSTANCE_H
+#define LLVM_FLANG_FRONTEND_COMPILERINSTANCE_H
+
+#include "flang/Frontend/CompilerInvocation.h"
+
+#include <cassert>
+#include <memory>
+
+namespace Fortran::frontend {
+
+class CompilerInstance {
+
+  /// The options used in this compiler instance.
+  std::shared_ptr<CompilerInvocation> invocation_;
+
+  /// The diagnostics engine instance.
+  llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_;
+
+public:
+  explicit CompilerInstance();
+
+  ~CompilerInstance();
+  CompilerInvocation &GetInvocation() {
+    assert(invocation_ && "Compiler instance has no invocation!");
+    return *invocation_;
+  };
+
+  /// }
+  /// @name Forwarding Methods
+  /// {
+
+  clang::DiagnosticOptions &GetDiagnosticOpts() {
+    return invocation_->GetDiagnosticOpts();
+  }
+  const clang::DiagnosticOptions &GetDiagnosticOpts() const {
+    return invocation_->GetDiagnosticOpts();
+  }
+
+  FrontendOptions &GetFrontendOpts() { return invocation_->GetFrontendOpts(); }
+  const FrontendOptions &GetFrontendOpts() const {
+    return invocation_->GetFrontendOpts();
+  }
+
+  /// }
+  /// @name Diagnostics Engine
+  /// {
+
+  bool HasDiagnostics() const { return diagnostics_ != nullptr; }
+
+  /// Get the current diagnostics engine.
+  clang::DiagnosticsEngine &GetDiagnostics() const {
+    assert(diagnostics_ && "Compiler instance has no diagnostics!");
+    return *diagnostics_;
+  }
+
+  /// SetDiagnostics - Replace the current diagnostics engine.
+  void SetDiagnostics(clang::DiagnosticsEngine *value);
+
+  clang::DiagnosticConsumer &GetDiagnosticClient() const {
+    assert(diagnostics_ && diagnostics_->getClient() &&
+        "Compiler instance has no diagnostic client!");
+    return *diagnostics_->getClient();
+  }
+
+  /// Get the current diagnostics engine.
+  clang::DiagnosticsEngine &getDiagnostics() const {
+    assert(diagnostics_ && "Compiler instance has no diagnostics!");
+    return *diagnostics_;
+  }
+
+  /// }
+  /// @name Construction Utility Methods
+  /// {
+
+  /// Create a DiagnosticsEngine object with a the TextDiagnosticPrinter.
+  ///
+  /// If no diagnostic client is provided, this creates a
+  /// DiagnosticConsumer that is owned by the returned diagnostic
+  /// object, if using directly the caller is responsible for
+  /// releasing the returned DiagnosticsEngine's client eventually.
+  ///
+  /// \param opts - The diagnostic options; note that the created text
+  /// diagnostic object contains a reference to these options.
+  ///
+  /// \param client If non-NULL, a diagnostic client that will be
+  /// attached to (and, then, owned by) the returned DiagnosticsEngine
+  /// object.
+  ///
+  /// \return The new object on success, or null on failure.
+  static clang::IntrusiveRefCntPtr<clang::DiagnosticsEngine> CreateDiagnostics(
+      clang::DiagnosticOptions *opts,
+      clang::DiagnosticConsumer *client = nullptr, bool shouldOwnClient = true);
+  void CreateDiagnostics(
+      clang::DiagnosticConsumer *client = nullptr, bool shouldOwnClient = true);
+};
+
+} // end namespace Fortran::frontend
+#endif // LLVM_FLANG_FRONTEND_COMPILERINSTANCE_H