[clang][cas] Create explicit IncludeTree::ModuleMap

Instead of putting the modulemap file in the include-tree filesystem and
parsing it at build time, create a data structure that represents just
the parts of the modulemap we need when building the module:

* Flags (explicit, system, framework, etc.)
* Exports (export *, export Foo)
* LinkLibraries (implicit framework autolink, link Foo)

Additionally, we add modular headers lazily by inserting known headers
when we encounter an include-tree header that is part of a submodule
(this is needed for missing #include diagnostics).

This removes the possibility of mismatches between header paths seen
during modulemap parsing from the paths we embed due to #includes, for
exmaple due to VFS virtual vs external paths.

(cherry picked from commit a237e68)

 Conflicts:
	clang/include/clang/Basic/Module.h
	clang/lib/Serialization/ASTWriter.cpp

Author: benlangmuir

clang/include/clang/Basic/DiagnosticFrontendKinds.td

@@ -228,6 +228,8 @@ def err_missing_module_name : Error<
   DefaultFatal;
 def err_missing_module : Error<
   "no module named '%0' declared in module map file '%1'">, DefaultFatal;
+def err_missing_module_include_tree : Error<
+  "no module named '%0' declared in include-tree module map '%1'">, DefaultFatal;
 def err_no_submodule : Error<"no submodule named %0 in module '%1'">;
 def err_no_submodule_suggest : Error<
   "no submodule named %0 in module '%1'; did you mean '%2'?">;

clang/include/clang/Basic/Module.h

@@ -111,6 +111,10 @@ class alignas(8) Module {
     /// of header files.
     ModuleMapModule,
 
+    /// This is a module that was defined by a module map and built out
+    /// of header files as part of an \c IncludeTree.
+    IncludeTreeModuleMap,
+
     /// This is a C++ 20 header unit.
     ModuleHeaderUnit,
 
@@ -204,7 +208,9 @@ class alignas(8) Module {
 
   bool isPrivateModule() const { return Kind == PrivateModuleFragment; }
 
-  bool isModuleMapModule() const { return Kind == ModuleMapModule; }
+  bool isModuleMapModule() const {
+    return Kind == ModuleMapModule || Kind == IncludeTreeModuleMap;
+  }
 
 private:
   /// The submodules of this module, indexed by name.

clang/include/clang/CAS/IncludeTree.h

@@ -54,7 +54,9 @@ class IncludeTree : public IncludeTreeBase<IncludeTree> {
   class File;
   class FileList;
   class Node;
+  class Module;
   class ModuleImport;
+  class ModuleMap;
 
   Expected<File> getBaseFile();
 
@@ -381,6 +383,262 @@ class IncludeTree::Node {
   NodeKind K;
 };
 
+/// Module or submodule declaration as part of the \c IncludeTreeRoot module
+/// map structure.
+class IncludeTree::Module : public IncludeTreeBase<Module> {
+public:
+  static constexpr StringRef getNodeKind() { return "Modu"; }
+
+  class ExportList;
+  class LinkLibraryList;
+
+  struct ModuleFlags {
+    bool IsFramework : 1;
+    bool IsExplicit : 1;
+    bool IsExternC : 1;
+    bool IsSystem : 1;
+    ModuleFlags()
+        : IsFramework(false), IsExplicit(false), IsExternC(false),
+          IsSystem(false) {}
+  };
+
+  ModuleFlags getFlags() const;
+
+  /// The name of the current (sub)module.
+  StringRef getName() const { return dataAfterFlags(); }
+
+  size_t getNumSubmodules() const;
+
+  Expected<Module> getSubmodule(size_t I) {
+    assert(I < getNumSubmodules());
+    auto Node = getCAS().getProxy(getReference(I));
+    if (!Node)
+      return Node.takeError();
+    return Module(*Node);
+  }
+
+  llvm::Error forEachSubmodule(llvm::function_ref<llvm::Error(Module)> CB);
+
+  std::optional<ObjectRef> getExportsRef() const {
+    if (std::optional<unsigned> Index = getExportsIndex())
+      return getReference(*Index);
+    return std::nullopt;
+  }
+  std::optional<ObjectRef> getLinkLibrariesRef() const {
+    if (std::optional<unsigned> Index = getLinkLibrariesIndex())
+      return getReference(*Index);
+    return std::nullopt;
+  }
+
+  /// The list of modules that this submodule re-exports.
+  Expected<std::optional<ExportList>> getExports();
+
+  /// The list of modules that this submodule re-exports.
+  Expected<std::optional<LinkLibraryList>> getLinkLibraries();
+
+  llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0);
+
+  static Expected<Module> create(ObjectStore &DB, StringRef ModuleName,
+                                 ModuleFlags Flags,
+                                 ArrayRef<ObjectRef> Submodules,
+                                 std::optional<ObjectRef> ExportList,
+                                 std::optional<ObjectRef> LinkLibraries);
+
+  static bool isValid(const ObjectProxy &Node) {
+    if (!IncludeTreeBase::isValid(Node))
+      return false;
+    IncludeTreeBase Base(Node);
+    return Base.getData().size() > 1;
+  }
+  static bool isValid(ObjectStore &DB, ObjectRef Ref) {
+    auto Node = DB.getProxy(Ref);
+    if (!Node) {
+      llvm::consumeError(Node.takeError());
+      return false;
+    }
+    return isValid(*Node);
+  }
+
+private:
+  char rawFlags() const { return getData()[0]; }
+  StringRef dataAfterFlags() const { return getData().drop_front(); }
+  bool hasExports() const;
+  bool hasLinkLibraries() const;
+  std::optional<unsigned> getExportsIndex() const;
+  std::optional<unsigned> getLinkLibrariesIndex() const;
+
+  explicit Module(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) {
+    assert(isValid(*this));
+  }
+
+  friend class IncludeTreeBase<Module>;
+  friend class IncludeTreeRoot;
+  friend class ModuleMap;
+};
+
+/// The set of modules re-exported by another module.
+class IncludeTree::Module::ExportList : public IncludeTreeBase<ExportList> {
+public:
+  static constexpr StringRef getNodeKind() { return "ExpL"; }
+
+  /// An explicit export.
+  struct Export {
+    StringRef ModuleName;
+    bool Wildcard;
+  };
+
+  /// Whether this module exports all imported modules (`export *`).
+  bool hasGlobalWildcard() const;
+
+  size_t getNumExplicitExports() const { return getNumReferences(); }
+
+  /// Whether the explicit export at \p I has a wildcard (`export MyModule.*`).
+  bool exportHasWildcard(size_t I) const;
+
+  Expected<Export> getExplicitExport(size_t I);
+
+  /// Calls \p CB for each explicit export declaration.
+  llvm::Error forEachExplicitExport(llvm::function_ref<llvm::Error(Export)> CB);
+
+  llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0);
+
+  static Expected<ExportList> create(ObjectStore &DB, ArrayRef<Export> Exports,
+                                     bool GlobalWildcard);
+
+  static bool isValid(const ObjectProxy &Node) {
+    if (!IncludeTreeBase::isValid(Node))
+      return false;
+    IncludeTreeBase Base(Node);
+    size_t ExpectedBitSize = Base.getNumReferences() + 1;
+    size_t ExpectedSize =
+        ExpectedBitSize / 8 + (ExpectedBitSize % 8 == 0 ? 0 : 1);
+    return Base.getData().size() == ExpectedSize;
+  }
+  static bool isValid(ObjectStore &DB, ObjectRef Ref) {
+    auto Node = DB.getProxy(Ref);
+    if (!Node) {
+      llvm::consumeError(Node.takeError());
+      return false;
+    }
+    return isValid(*Node);
+  }
+
+private:
+  explicit ExportList(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) {
+    assert(isValid(*this));
+  }
+
+  friend class IncludeTreeBase<ExportList>;
+  friend class Module;
+};
+
+/// The set of libraries to link against when using a module.
+class IncludeTree::Module::LinkLibraryList
+    : public IncludeTreeBase<LinkLibraryList> {
+public:
+  static constexpr StringRef getNodeKind() { return "LnkL"; }
+
+  struct LinkLibrary {
+    StringRef Library;
+    bool IsFramework;
+  };
+
+  size_t getNumLibraries() const { return getNumReferences(); }
+
+  /// Whether the library at \p I is a framework.
+  bool isFramework(size_t I) const;
+
+  ObjectRef getLibraryNameRef(size_t I) const { return getReference(I); }
+
+  Expected<LinkLibrary> getLinkLibrary(size_t I) {
+    auto Name = getCAS().getProxy(getLibraryNameRef(I));
+    if (!Name)
+      return Name.takeError();
+    return LinkLibrary{Name->getData(), isFramework(I)};
+  }
+
+  /// Calls \p CB for each link libary.
+  llvm::Error
+  forEachLinkLibrary(llvm::function_ref<llvm::Error(LinkLibrary)> CB);
+
+  llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0);
+
+  static Expected<LinkLibraryList> create(ObjectStore &DB,
+                                          ArrayRef<LinkLibrary> Exports);
+
+  static bool isValid(const ObjectProxy &Node) {
+    if (!IncludeTreeBase::isValid(Node))
+      return false;
+    IncludeTreeBase Base(Node);
+    size_t ExpectedBitSize = Base.getNumReferences();
+    size_t ExpectedSize =
+        ExpectedBitSize / 8 + (ExpectedBitSize % 8 == 0 ? 0 : 1);
+    return Base.getData().size() == ExpectedSize;
+  }
+  static bool isValid(ObjectStore &DB, ObjectRef Ref) {
+    auto Node = DB.getProxy(Ref);
+    if (!Node) {
+      llvm::consumeError(Node.takeError());
+      return false;
+    }
+    return isValid(*Node);
+  }
+
+private:
+  explicit LinkLibraryList(ObjectProxy Node)
+      : IncludeTreeBase(std::move(Node)) {
+    assert(isValid(*this));
+  }
+
+  friend class IncludeTreeBase<LinkLibraryList>;
+  friend class Module;
+};
+
+class IncludeTree::ModuleMap : public IncludeTreeBase<ModuleMap> {
+public:
+  static constexpr StringRef getNodeKind() { return "ModM"; }
+
+  size_t getNumModules() const { return getNumReferences(); }
+
+  Expected<Module> getModule(size_t I) {
+    auto Node = getCAS().getProxy(getReference(I));
+    if (!Node)
+      return Node.takeError();
+    return Module(*Node);
+  }
+
+  /// Calls \p CB for each module declaration.
+  llvm::Error forEachModule(llvm::function_ref<llvm::Error(Module)> CB);
+
+  llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0);
+
+  static Expected<ModuleMap> create(ObjectStore &DB,
+                                    ArrayRef<ObjectRef> Modules);
+
+  static bool isValid(const ObjectProxy &Node) {
+    if (!IncludeTreeBase::isValid(Node))
+      return false;
+    IncludeTreeBase Base(Node);
+    return Base.getData().empty();
+  }
+  static bool isValid(ObjectStore &DB, ObjectRef Ref) {
+    auto Node = DB.getProxy(Ref);
+    if (!Node) {
+      llvm::consumeError(Node.takeError());
+      return false;
+    }
+    return isValid(*Node);
+  }
+
+private:
+  explicit ModuleMap(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) {
+    assert(isValid(*this));
+  }
+
+  friend class IncludeTreeBase<ModuleMap>;
+  friend class IncludeTreeRoot;
+};
+
 /// Represents the include-tree result for a translation unit.
 class IncludeTreeRoot : public IncludeTreeBase<IncludeTreeRoot> {
 public:
@@ -426,12 +684,12 @@ class IncludeTreeRoot : public IncludeTreeBase<IncludeTreeRoot> {
     return std::nullopt;
   }
 
-  Expected<std::optional<IncludeTree::File>> getModuleMapFile() {
+  Expected<std::optional<IncludeTree::ModuleMap>> getModuleMap() {
     if (std::optional<ObjectRef> Ref = getModuleMapRef()) {
       auto Node = getCAS().getProxy(*Ref);
       if (!Node)
         return Node.takeError();
-      return IncludeTree::File(*Node);
+      return IncludeTree::ModuleMap(*Node);
     }
     return std::nullopt;
   }

clang/include/clang/Frontend/FrontendOptions.h

@@ -246,6 +246,10 @@ class FrontendInputFile {
                     bool IsSystem = false)
       : File(File.str()), IncludeTree(std::move(Tree)), Kind(Kind),
         IsSystem(IsSystem) {}
+  FrontendInputFile(cas::ObjectRef Tree, llvm::MemoryBufferRef Buffer,
+                    InputKind Kind, bool IsSystem = false)
+      : Buffer(Buffer), IncludeTree(std::move(Tree)), Kind(Kind),
+        IsSystem(IsSystem) {}
 
   InputKind getKind() const { return Kind; }
   bool isSystem() const { return IsSystem; }

clang/lib/AST/Decl.cpp

@@ -1596,6 +1596,7 @@ Module *Decl::getOwningModuleForLinkage(bool IgnoreLinkage) const {
 
   switch (M->Kind) {
   case Module::ModuleMapModule:
+  case Module::IncludeTreeModuleMap:
     // Module map modules have no special linkage semantics.
     return nullptr;