[clang][deps] Cherry-pick recent upstream commits

clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h

@@ -178,7 +178,11 @@ class ModuleDepCollectorPP final : public PPCallbacks {
 
   /// Adds direct modular dependencies that have already been built to the
   /// ModuleDeps instance.
-  void addDirectPrebuiltModuleDeps(const Module *M, ModuleDeps &MD);
+  void
+  addAllSubmodulePrebuiltDeps(const Module *M, ModuleDeps &MD,
+                              llvm::DenseSet<const Module *> &SeenSubmodules);
+  void addModulePrebuiltDeps(const Module *M, ModuleDeps &MD,
+                             llvm::DenseSet<const Module *> &SeenSubmodules);
 
   /// Traverses the previously collected direct modular dependencies to discover
   /// transitive modular dependencies and fills the parent \c ModuleDepCollector

clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp

@@ -180,7 +180,7 @@ DependencyScanningTool::getFullDependencies(
   private:
     std::vector<std::string> Dependencies;
     std::vector<PrebuiltModuleDep> PrebuiltModuleDeps;
-    std::unordered_map<std::string, ModuleDeps> ClangModuleDeps;
+    std::map<std::string, ModuleDeps> ClangModuleDeps;
     std::string ContextHash;
     std::vector<std::string> OutputPaths;
     const llvm::StringSet<> &AlreadySeen;

clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp

@@ -23,6 +23,9 @@ CompilerInvocation ModuleDepCollector::makeInvocationForModuleBuildWithoutPaths(
   // Make a deep copy of the original Clang invocation.
   CompilerInvocation CI(OriginalInvocation);
 
+  CI.getLangOpts()->resetNonModularOptions();
+  CI.getPreprocessorOpts().resetNonModularOptions();
+
   // Remove options incompatible with explicit module build.
   CI.getFrontendOpts().Inputs.clear();
   CI.getFrontendOpts().OutputFile.clear();
@@ -39,8 +42,6 @@ CompilerInvocation ModuleDepCollector::makeInvocationForModuleBuildWithoutPaths(
     CI.getFrontendOpts().ModuleMapFiles.push_back(PrebuiltModule.ModuleMapFile);
   }
 
-  CI.getPreprocessorOpts().ImplicitPCHInclude.clear();
-
   return CI;
 }
 
@@ -249,7 +250,8 @@ ModuleID ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
   // Add direct prebuilt module dependencies now, so that we can use them when
   // creating a CompilerInvocation and computing context hash for this
   // ModuleDeps instance.
-  addDirectPrebuiltModuleDeps(M, MD);
+  llvm::DenseSet<const Module *> SeenModules;
+  addAllSubmodulePrebuiltDeps(M, MD, SeenModules);
 
   MD.Invocation = MDC.makeInvocationForModuleBuildWithoutPaths(MD);
   MD.ID.ContextHash = MD.Invocation.getModuleHash(Instance.getDiagnostics());
@@ -260,12 +262,23 @@ ModuleID ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
   return MD.ID;
 }
 
-void ModuleDepCollectorPP::addDirectPrebuiltModuleDeps(const Module *M,
-                                                       ModuleDeps &MD) {
+void ModuleDepCollectorPP::addAllSubmodulePrebuiltDeps(
+    const Module *M, ModuleDeps &MD,
+    llvm::DenseSet<const Module *> &SeenSubmodules) {
+  addModulePrebuiltDeps(M, MD, SeenSubmodules);
+
+  for (const Module *SubM : M->submodules())
+    addAllSubmodulePrebuiltDeps(SubM, MD, SeenSubmodules);
+}
+
+void ModuleDepCollectorPP::addModulePrebuiltDeps(
+    const Module *M, ModuleDeps &MD,
+    llvm::DenseSet<const Module *> &SeenSubmodules) {
   for (const Module *Import : M->Imports)
     if (Import->getTopLevelModule() != M->getTopLevelModule())
-      if (MDC.isPrebuiltModule(Import))
-        MD.PrebuiltModuleDeps.emplace_back(Import);
+      if (MDC.isPrebuiltModule(Import->getTopLevelModule()))
+        if (SeenSubmodules.insert(Import->getTopLevelModule()).second)
+          MD.PrebuiltModuleDeps.emplace_back(Import->getTopLevelModule());
 }
 
 void ModuleDepCollectorPP::addAllSubmoduleDeps(

clang/test/ClangScanDeps/Inputs/modules-pch-common-submodule/cdb_pch.json

@@ -0,0 +1,7 @@
+[
+  {
+    "directory": "DIR",
+    "command": "clang -x c-header DIR/pch.h -fmodules -gmodules -fimplicit-module-maps -fmodules-cache-path=DIR/cache -o DIR/pch.h.gch",
+    "file": "DIR/pch.h"
+  }
+]

clang/test/ClangScanDeps/Inputs/modules-pch-common-submodule/cdb_tu.json

@@ -0,0 +1,7 @@
+[
+  {
+    "directory": "DIR",
+    "command": "clang -fsyntax-only DIR/tu.c -fmodules -gmodules -fimplicit-module-maps -fmodules-cache-path=DIR/cache -include DIR/pch.h -o DIR/tu.o",
+    "file": "DIR/tu.c"
+  }
+]

clang/test/ClangScanDeps/Inputs/modules-pch-common-submodule/mod_tu.h

@@ -0,0 +1 @@
+#include "mod_common_sub.h"

clang/test/ClangScanDeps/Inputs/modules-pch-common-submodule/module.modulemap

@@ -0,0 +1,11 @@
+module ModCommon {
+  header "mod_common.h"
+
+  module ModCommonSub {
+    header "mod_common_sub.h"
+  }
+}
+
+module ModTU {
+  header "mod_tu.h"
+}

clang/test/ClangScanDeps/Inputs/modules-pch-common-submodule/pch.h

@@ -0,0 +1 @@
+#include "mod_common.h"

clang/test/ClangScanDeps/Inputs/modules-pch-common-submodule/tu.c

@@ -0,0 +1 @@
+#include "mod_tu.h"

clang/test/ClangScanDeps/Inputs/modules-pch-common-via-submodule/cdb_pch.json

@@ -0,0 +1,7 @@
+[
+  {
+    "directory": "DIR",
+    "command": "clang -x c-header DIR/pch.h -fmodules -gmodules -fimplicit-module-maps -fmodules-cache-path=DIR/cache -o DIR/pch.h.gch",
+    "file": "DIR/pch.h"
+  }
+]

clang/test/ClangScanDeps/Inputs/modules-pch-common-via-submodule/cdb_tu.json

@@ -0,0 +1,7 @@
+[
+  {
+    "directory": "DIR",
+    "command": "clang -fsyntax-only DIR/tu.c -fmodules -gmodules -fimplicit-module-maps -fmodules-cache-path=DIR/cache -include DIR/pch.h -o DIR/tu.o",
+    "file": "DIR/tu.c"
+  }
+]

clang/test/ClangScanDeps/Inputs/modules-pch-common-via-submodule/mod_tu_sub.h

@@ -0,0 +1 @@
+#include "mod_common.h"

clang/test/ClangScanDeps/Inputs/modules-pch-common-via-submodule/module.modulemap

@@ -0,0 +1,11 @@
+module ModCommon {
+  header "mod_common.h"
+}
+
+module ModTU {
+  header "mod_tu.h"
+
+  module ModTUSub {
+    header "mod_tu_sub.h"
+  }
+}

clang/test/ClangScanDeps/Inputs/modules-pch-common-via-submodule/pch.h

@@ -0,0 +1 @@
+#include "mod_common.h"

clang/test/ClangScanDeps/Inputs/modules-pch-common-via-submodule/tu.c

@@ -0,0 +1 @@
+#include "mod_tu.h"

clang/test/ClangScanDeps/Inputs/removed-args/cdb.json.template

@@ -0,0 +1,7 @@
+[
+  {
+    "directory": "DIR",
+    "command": "clang -fsyntax-only DIR/tu.c -fmodules -fimplicit-module-maps -fmodules-cache-path=DIR/cache -include DIR/header.h -o DIR/tu.o",
+    "file": "DIR/tu.c"
+  }
+]

clang/test/ClangScanDeps/Inputs/removed-args/header.h

@@ -0,0 +1 @@
+#include "mod_header.h"

clang/test/ClangScanDeps/Inputs/removed-args/module.modulemap

@@ -0,0 +1,2 @@
+module ModHeader { header "mod_header.h" }
+module ModTU     { header "mod_tu.h"     }

clang/test/ClangScanDeps/Inputs/removed-args/tu.c

@@ -0,0 +1 @@
+#include "mod_tu.h"

clang/test/ClangScanDeps/modules-pch-common-submodule.c

@@ -0,0 +1,140 @@
+// Check that when depending on a precompiled module, we depend on the
+// **top-level** module. Submodules don't have some information present (for
+// example the path to the modulemap file) and depending on them might cause
+// problems in the dependency scanner (e.g. generating empty `-fmodule-map-file=`
+// arguments).
+
+// RUN: rm -rf %t && mkdir %t
+// RUN: cp %S/Inputs/modules-pch-common-submodule/* %t
+
+// Scan dependencies of the PCH:
+//
+// RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch-common-submodule/cdb_pch.json > %t/cdb.json
+// RUN: echo -%t > %t/result_pch.json
+// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full \
+// RUN:   -generate-modules-path-args -module-files-dir %t/build >> %t/result_pch.json
+// RUN: cat %t/result_pch.json | sed 's:\\\\\?:/:g' | FileCheck %s -check-prefix=CHECK-PCH
+//
+// CHECK-PCH:      -[[PREFIX:.*]]
+// CHECK-PCH-NEXT: {
+// CHECK-PCH-NEXT:   "modules": [
+// CHECK-PCH-NEXT:     {
+// CHECK-PCH-NEXT:       "clang-module-deps": [],
+// CHECK-PCH-NEXT:       "clang-modulemap-file": "[[PREFIX]]/module.modulemap",
+// CHECK-PCH-NEXT:       "command-line": [
+// CHECK-PCH-NEXT:         "-cc1"
+// CHECK-PCH:              "-emit-module"
+// CHECK-PCH:              "-fmodules"
+// CHECK-PCH:              "-fmodule-name=ModCommon"
+// CHECK-PCH:              "-fno-implicit-modules"
+// CHECK-PCH:            ],
+// CHECK-PCH-NEXT:       "context-hash": "[[HASH_MOD_COMMON:.*]]",
+// CHECK-PCH-NEXT:       "file-deps": [
+// CHECK-PCH-NEXT:         "[[PREFIX]]/mod_common.h",
+// CHECK-PCH-NEXT:         "[[PREFIX]]/mod_common_sub.h",
+// CHECK-PCH-NEXT:         "[[PREFIX]]/module.modulemap"
+// CHECK-PCH-NEXT:       ],
+// CHECK-PCH-NEXT:       "name": "ModCommon"
+// CHECK-PCH-NEXT:     }
+// CHECK-PCH-NEXT:   ],
+// CHECK-PCH-NEXT:   "translation-units": [
+// CHECK-PCH-NEXT:     {
+// CHECK-PCH-NEXT:       "clang-context-hash": "[[HASH_PCH:.*]]",
+// CHECK-PCH-NEXT:       "clang-module-deps": [
+// CHECK-PCH-NEXT:         {
+// CHECK-PCH-NEXT:           "context-hash": "[[HASH_MOD_COMMON]]",
+// CHECK-PCH-NEXT:           "module-name": "ModCommon"
+// CHECK-PCH-NEXT:         }
+// CHECK-PCH-NEXT:       ],
+// CHECK-PCH-NEXT:       "command-line": [
+// CHECK-PCH-NEXT:         "-fno-implicit-modules"
+// CHECK-PCH-NEXT:         "-fno-implicit-module-maps"
+// CHECK-PCH-NEXT:         "-fmodule-file=[[PREFIX]]/build/[[HASH_MOD_COMMON]]/ModCommon-{{.*}}.pcm"
+// CHECK-PCH-NEXT:         "-fmodule-map-file=[[PREFIX]]/module.modulemap"
+// CHECK-PCH-NEXT:       ],
+// CHECK-PCH-NEXT:       "file-deps": [
+// CHECK-PCH-NEXT:         "[[PREFIX]]/pch.h"
+// CHECK-PCH-NEXT:       ],
+// CHECK-PCH-NEXT:       "input-file": "[[PREFIX]]/pch.h"
+// CHECK-PCH-NEXT:     }
+// CHECK-PCH-NEXT:   ]
+// CHECK-PCH-NEXT: }
+
+// Explicitly build the PCH:
+//
+// RUN: tail -n +2 %t/result_pch.json > %t/result_pch_stripped.json
+// RUN: %python %S/../../utils/module-deps-to-rsp.py %t/result_pch_stripped.json \
+// RUN:   --module-name=ModCommon > %t/mod_common.cc1.rsp
+// RUN: %python %S/../../utils/module-deps-to-rsp.py %t/result_pch_stripped.json \
+// RUN:   --tu-index=0 > %t/pch.rsp
+//
+// RUN: %clang @%t/mod_common.cc1.rsp
+// RUN: %clang -x c-header %t/pch.h -fmodules -gmodules -fimplicit-module-maps \
+// RUN:   -fmodules-cache-path=%t/cache -o %t/pch.h.gch @%t/pch.rsp
+
+// Scan dependencies of the TU:
+//
+// RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch-common-submodule/cdb_tu.json > %t/cdb.json
+// RUN: echo -%t > %t/result_tu.json
+// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full \
+// RUN:   -generate-modules-path-args -module-files-dir %t/build >> %t/result_tu.json
+// RUN: cat %t/result_tu.json | sed 's:\\\\\?:/:g' | FileCheck %s -check-prefix=CHECK-TU
+//
+// CHECK-TU:      -[[PREFIX:.*]]
+// CHECK-TU-NEXT: {
+// CHECK-TU-NEXT:   "modules": [
+// CHECK-TU-NEXT:     {
+// CHECK-TU-NEXT:       "clang-module-deps": [],
+// CHECK-TU-NEXT:       "clang-modulemap-file": "[[PREFIX]]/module.modulemap",
+// CHECK-TU-NEXT:       "command-line": [
+// CHECK-TU-NEXT:         "-cc1"
+// CHECK-TU:              "-fmodule-map-file=[[PREFIX]]/module.modulemap"
+// CHECK-TU:              "-emit-module"
+// CHECK-TU:              "-fmodule-file=[[PREFIX]]/build/[[HASH_MOD_COMMON:.*]]/ModCommon-{{.*}}.pcm"
+// CHECK-TU:              "-fmodules"
+// CHECK-TU:              "-fmodule-name=ModTU"
+// CHECK-TU:              "-fno-implicit-modules"
+// CHECK-TU:            ],
+// CHECK-TU-NEXT:       "context-hash": "[[HASH_MOD_TU:.*]]",
+// CHECK-TU-NEXT:       "file-deps": [
+// CHECK-TU-NEXT:         "[[PREFIX]]/mod_tu.h",
+// CHECK-TU-NEXT:         "[[PREFIX]]/module.modulemap"
+// CHECK-TU-NEXT:       ],
+// CHECK-TU-NEXT:       "name": "ModTU"
+// CHECK-TU-NEXT:     }
+// CHECK-TU-NEXT:   ],
+// CHECK-TU-NEXT:   "translation-units": [
+// CHECK-TU-NEXT:     {
+// CHECK-TU-NEXT:       "clang-context-hash": "[[HASH_TU:.*]]",
+// CHECK-TU-NEXT:       "clang-module-deps": [
+// CHECK-TU-NEXT:         {
+// CHECK-TU-NEXT:           "context-hash": "[[HASH_MOD_TU]]"
+// CHECK-TU-NEXT:           "module-name": "ModTU"
+// CHECK-TU-NEXT:         }
+// CHECK-TU-NEXT:       ],
+// CHECK-TU-NEXT:       "command-line": [
+// CHECK-TU-NEXT:         "-fno-implicit-modules",
+// CHECK-TU-NEXT:         "-fno-implicit-module-maps",
+// CHECK-TU-NEXT:         "-fmodule-file=[[PREFIX]]/build/[[HASH_MOD_TU:.*]]/ModTU-{{.*}}.pcm",
+// CHECK-TU-NEXT:         "-fmodule-map-file=[[PREFIX]]/module.modulemap"
+// CHECK-TU-NEXT:       ],
+// CHECK-TU-NEXT:       "file-deps": [
+// CHECK-TU-NEXT:         "[[PREFIX]]/tu.c",
+// CHECK-TU-NEXT:         "[[PREFIX]]/pch.h.gch"
+// CHECK-TU-NEXT:       ],
+// CHECK-TU-NEXT:       "input-file": "[[PREFIX]]/tu.c"
+// CHECK-TU-NEXT:     }
+// CHECK-TU-NEXT:   ]
+// CHECK-TU-NEXT: }
+
+// Explicitly build the TU:
+//
+// RUN: tail -n +2 %t/result_tu.json > %t/result_tu_stripped.json
+// RUN: %python %S/../../utils/module-deps-to-rsp.py %t/result_tu_stripped.json \
+// RUN:   --module-name=ModTU > %t/mod_tu.cc1.rsp
+// RUN: %python %S/../../utils/module-deps-to-rsp.py %t/result_tu_stripped.json \
+// RUN:   --tu-index=0 > %t/tu.rsp
+//
+// RUN: %clang @%t/mod_tu.cc1.rsp
+// RUN: %clang -fsyntax-only %t/tu.c -fmodules -gmodules -fimplicit-module-maps \
+// RUN:   -fmodules-cache-path=%t/cache -include %t/pch.h -o %t/tu.o @%t/tu.rsp