[6.2][Clang] Cherry-pick changes for header and module import tracing

clang/include/clang/Basic/DiagnosticDriverKinds.td

@@ -399,8 +399,14 @@ def err_drv_print_header_env_var : Error<
   "environment variable CC_PRINT_HEADERS_%select{FORMAT|FILTERING}0 has invalid value %1">;
 def err_drv_print_header_env_var_combination : Error<
   "unsupported combination: CC_PRINT_HEADERS_FORMAT=%0 and CC_PRINT_HEADERS_FILTERING=%1">;
-def err_drv_print_header_env_var_combination_cc1 : Error<
+def err_drv_print_header_env_var_invalid_format : Error<
+  "environment variable CC_PRINT_HEADERS_FORMAT=%0 requires a compatible value for CC_PRINT_HEADERS_FILTERING">;
+def err_drv_print_header_cc1_invalid_combination : Error<
   "unsupported combination: -header-include-format=%0 and -header-include-filtering=%1">;
+def err_drv_print_header_cc1_invalid_filtering : Error<
+  "-header-include-filtering=%0 requires a compatible value for -header-include-format">;
+def err_drv_print_header_cc1_invalid_format : Error<
+  "-header-include-format=%0 requires a compatible value for -header-include-filtering">;
 
 def warn_O4_is_O3 : Warning<"-O4 is equivalent to -O3">, InGroup<Deprecated>;
 def warn_drv_optimization_value : Warning<"optimization level '%0' is not supported; using '%1%2' instead">,

clang/include/clang/Basic/HeaderInclude.h

@@ -23,8 +23,14 @@ enum HeaderIncludeFormatKind { HIFMT_None, HIFMT_Textual, HIFMT_JSON };
 
 /// Whether header information is filtered or not. If HIFIL_Only_Direct_System
 /// is used, only information on system headers directly included from
-/// non-system headers is emitted.
-enum HeaderIncludeFilteringKind { HIFIL_None, HIFIL_Only_Direct_System };
+/// non-system files is emitted. The HIFIL_Direct_Per_File filtering shows the
+/// direct imports and includes for each non-system source and header file
+/// separately.
+enum HeaderIncludeFilteringKind {
+  HIFIL_None,
+  HIFIL_Only_Direct_System,
+  HIFIL_Direct_Per_File
+};
 
 inline HeaderIncludeFormatKind
 stringToHeaderIncludeFormatKind(const char *Str) {
@@ -40,6 +46,7 @@ inline bool stringToHeaderIncludeFiltering(const char *Str,
       llvm::StringSwitch<std::pair<bool, HeaderIncludeFilteringKind>>(Str)
           .Case("none", {true, HIFIL_None})
           .Case("only-direct-system", {true, HIFIL_Only_Direct_System})
+          .Case("direct-per-file", {true, HIFIL_Direct_Per_File})
           .Default({false, HIFIL_None});
   Kind = P.second;
   return P.first;
@@ -64,6 +71,8 @@ headerIncludeFilteringKindToString(HeaderIncludeFilteringKind K) {
     return "none";
   case HIFIL_Only_Direct_System:
     return "only-direct-system";
+  case HIFIL_Direct_Per_File:
+    return "direct-per-file";
   }
   llvm_unreachable("Unknown HeaderIncludeFilteringKind enum");
 }

clang/include/clang/Driver/Options.td

@@ -7586,7 +7586,8 @@ def header_include_format_EQ : Joined<["-"], "header-include-format=">,
   MarshallingInfoEnum<DependencyOutputOpts<"HeaderIncludeFormat">, "HIFMT_Textual">;
 def header_include_filtering_EQ : Joined<["-"], "header-include-filtering=">,
   HelpText<"set the flag that enables filtering header information">,
-  Values<"none,only-direct-system">, NormalizedValues<["HIFIL_None", "HIFIL_Only_Direct_System"]>,
+  Values<"none,only-direct-system,direct-per-file">,
+  NormalizedValues<["HIFIL_None", "HIFIL_Only_Direct_System", "HIFIL_Direct_Per_File"]>,
   MarshallingInfoEnum<DependencyOutputOpts<"HeaderIncludeFiltering">, "HIFIL_None">;
 def show_includes : Flag<["--"], "show-includes">,
   HelpText<"Print cl.exe style /showIncludes to stdout">;

clang/lib/Frontend/CompilerInvocation.cpp

@@ -2541,13 +2541,25 @@ static bool ParseDependencyOutputArgs(DependencyOutputOptions &Opts,
 
   // Check for invalid combinations of header-include-format
   // and header-include-filtering.
-  if ((Opts.HeaderIncludeFormat == HIFMT_Textual &&
-       Opts.HeaderIncludeFiltering != HIFIL_None) ||
-      (Opts.HeaderIncludeFormat == HIFMT_JSON &&
-       Opts.HeaderIncludeFiltering != HIFIL_Only_Direct_System))
-    Diags.Report(diag::err_drv_print_header_env_var_combination_cc1)
-        << Args.getLastArg(OPT_header_include_format_EQ)->getValue()
-        << Args.getLastArg(OPT_header_include_filtering_EQ)->getValue();
+  if (Opts.HeaderIncludeFormat == HIFMT_Textual &&
+      Opts.HeaderIncludeFiltering != HIFIL_None) {
+    if (Args.hasArg(OPT_header_include_format_EQ))
+      Diags.Report(diag::err_drv_print_header_cc1_invalid_combination)
+          << headerIncludeFormatKindToString(Opts.HeaderIncludeFormat)
+          << headerIncludeFilteringKindToString(Opts.HeaderIncludeFiltering);
+    else
+      Diags.Report(diag::err_drv_print_header_cc1_invalid_filtering)
+          << headerIncludeFilteringKindToString(Opts.HeaderIncludeFiltering);
+  } else if (Opts.HeaderIncludeFormat == HIFMT_JSON &&
+             Opts.HeaderIncludeFiltering == HIFIL_None) {
+    if (Args.hasArg(OPT_header_include_filtering_EQ))
+      Diags.Report(diag::err_drv_print_header_cc1_invalid_combination)
+          << headerIncludeFormatKindToString(Opts.HeaderIncludeFormat)
+          << headerIncludeFilteringKindToString(Opts.HeaderIncludeFiltering);
+    else
+      Diags.Report(diag::err_drv_print_header_cc1_invalid_format)
+          << headerIncludeFormatKindToString(Opts.HeaderIncludeFormat);
+  }
 
   return Diags.getNumErrors() == NumErrorsBefore;
 }

clang/lib/Frontend/HeaderIncludeGen.cpp

@@ -106,6 +106,49 @@ class HeaderIncludesJSONCallback : public PPCallbacks {
   void FileSkipped(const FileEntryRef &SkippedFile, const Token &FilenameTok,
                    SrcMgr::CharacteristicKind FileType) override;
 };
+
+/// A callback for emitting direct header and module usage information to a
+/// file in JSON. The output format is like HeaderIncludesJSONCallback but has
+/// an array of separate entries, one for each non-system source file used in
+/// the compilation showing only the direct includes and imports from that file.
+class HeaderIncludesDirectPerFileCallback : public PPCallbacks {
+  SourceManager &SM;
+  HeaderSearch &HSI;
+  raw_ostream *OutputFile;
+  bool OwnsOutputFile;
+  using DependencyMap = llvm::DenseMap<FileEntryRef, SmallVector<FileEntryRef>>;
+  DependencyMap Dependencies;
+
+public:
+  HeaderIncludesDirectPerFileCallback(const Preprocessor *PP,
+                                      raw_ostream *OutputFile_,
+                                      bool OwnsOutputFile_)
+      : SM(PP->getSourceManager()), HSI(PP->getHeaderSearchInfo()),
+        OutputFile(OutputFile_), OwnsOutputFile(OwnsOutputFile_) {}
+
+  ~HeaderIncludesDirectPerFileCallback() override {
+    if (OwnsOutputFile)
+      delete OutputFile;
+  }
+
+  HeaderIncludesDirectPerFileCallback(
+      const HeaderIncludesDirectPerFileCallback &) = delete;
+  HeaderIncludesDirectPerFileCallback &
+  operator=(const HeaderIncludesDirectPerFileCallback &) = delete;
+
+  void EndOfMainFile() override;
+
+  void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
+                          StringRef FileName, bool IsAngled,
+                          CharSourceRange FilenameRange,
+                          OptionalFileEntryRef File, StringRef SearchPath,
+                          StringRef RelativePath, const Module *SuggestedModule,
+                          bool ModuleImported,
+                          SrcMgr::CharacteristicKind FileType) override;
+
+  void moduleImport(SourceLocation ImportLoc, ModuleIdPath Path,
+                    const Module *Imported) override;
+};
 }
 
 static void PrintHeaderInfo(raw_ostream *OutputFile, StringRef Filename,
@@ -192,14 +235,21 @@ void clang::AttachHeaderIncludeGen(Preprocessor &PP,
         MSStyle));
     break;
   }
-  case HIFMT_JSON: {
-    assert(DepOpts.HeaderIncludeFiltering == HIFIL_Only_Direct_System &&
-           "only-direct-system is the only option for filtering");
-    PP.addPPCallbacks(std::make_unique<HeaderIncludesJSONCallback>(
-        &PP, OutputFile, OwnsOutputFile));
+  case HIFMT_JSON:
+    switch (DepOpts.HeaderIncludeFiltering) {
+    default:
+      llvm_unreachable("Unknown HeaderIncludeFilteringKind enum");
+    case HIFIL_Only_Direct_System:
+      PP.addPPCallbacks(std::make_unique<HeaderIncludesJSONCallback>(
+          &PP, OutputFile, OwnsOutputFile));
+      break;
+    case HIFIL_Direct_Per_File:
+      PP.addPPCallbacks(std::make_unique<HeaderIncludesDirectPerFileCallback>(
+          &PP, OutputFile, OwnsOutputFile));
+      break;
+    }
     break;
   }
-  }
 }
 
 void HeaderIncludesCallback::FileChanged(SourceLocation Loc,
@@ -260,8 +310,11 @@ void HeaderIncludesCallback::FileSkipped(const FileEntryRef &SkippedFile, const
 
 void HeaderIncludesJSONCallback::EndOfMainFile() {
   OptionalFileEntryRef FE = SM.getFileEntryRefForID(SM.getMainFileID());
-  SmallString<256> MainFile(FE->getName());
-  SM.getFileManager().makeAbsolutePath(MainFile);
+  SmallString<256> MainFile;
+  if (FE) {
+    MainFile += FE->getName();
+    SM.getFileManager().makeAbsolutePath(MainFile);
+  }
 
   std::string Str;
   llvm::raw_string_ostream OS(Str);
@@ -319,3 +372,81 @@ void HeaderIncludesJSONCallback::FileSkipped(
 
   IncludedHeaders.push_back(SkippedFile.getName().str());
 }
+
+void HeaderIncludesDirectPerFileCallback::EndOfMainFile() {
+  if (Dependencies.empty())
+    return;
+
+  // Sort the files so that the output does not depend on the DenseMap order.
+  SmallVector<FileEntryRef> SourceFiles;
+  for (auto F = Dependencies.begin(), FEnd = Dependencies.end(); F != FEnd;
+       ++F) {
+    SourceFiles.push_back(F->first);
+  }
+  llvm::sort(SourceFiles, [](const FileEntryRef &LHS, const FileEntryRef &RHS) {
+    return LHS.getUID() < RHS.getUID();
+  });
+
+  std::string Str;
+  llvm::raw_string_ostream OS(Str);
+  llvm::json::OStream JOS(OS);
+  JOS.array([&] {
+    for (auto S = SourceFiles.begin(), SE = SourceFiles.end(); S != SE; ++S) {
+      JOS.object([&] {
+        SmallVector<FileEntryRef> &Deps = Dependencies[*S];
+        JOS.attribute("source", S->getName().str());
+        JOS.attributeArray("includes", [&] {
+          for (unsigned I = 0, N = Deps.size(); I != N; ++I)
+            JOS.value(Deps[I].getName().str());
+        });
+      });
+    }
+  });
+  OS << "\n";
+
+  if (OutputFile->get_kind() == raw_ostream::OStreamKind::OK_FDStream) {
+    llvm::raw_fd_ostream *FDS = static_cast<llvm::raw_fd_ostream *>(OutputFile);
+    if (auto L = FDS->lock())
+      *OutputFile << Str;
+  } else
+    *OutputFile << Str;
+}
+
+void HeaderIncludesDirectPerFileCallback::InclusionDirective(
+    SourceLocation HashLoc, const Token &IncludeTok, StringRef FileName,
+    bool IsAngled, CharSourceRange FilenameRange, OptionalFileEntryRef File,
+    StringRef SearchPath, StringRef RelativePath, const Module *SuggestedModule,
+    bool ModuleImported, SrcMgr::CharacteristicKind FileType) {
+  if (!File)
+    return;
+
+  SourceLocation Loc = SM.getExpansionLoc(HashLoc);
+  if (SM.isInSystemHeader(Loc))
+    return;
+  OptionalFileEntryRef FromFile = SM.getFileEntryRefForID(SM.getFileID(Loc));
+  if (!FromFile)
+    return;
+
+  Dependencies[*FromFile].push_back(*File);
+}
+
+void HeaderIncludesDirectPerFileCallback::moduleImport(SourceLocation ImportLoc,
+                                                       ModuleIdPath Path,
+                                                       const Module *Imported) {
+  if (!Imported)
+    return;
+
+  SourceLocation Loc = SM.getExpansionLoc(ImportLoc);
+  if (SM.isInSystemHeader(Loc))
+    return;
+  OptionalFileEntryRef FromFile = SM.getFileEntryRefForID(SM.getFileID(Loc));
+  if (!FromFile)
+    return;
+
+  OptionalFileEntryRef ModuleMapFile =
+      HSI.getModuleMap().getModuleMapFileForUniquing(Imported);
+  if (!ModuleMapFile)
+    return;
+
+  Dependencies[*FromFile].push_back(*ModuleMapFile);
+}

clang/test/Preprocessor/print-header-crash.modulemap

@@ -0,0 +1,2 @@
+// RUN: %clang_cc1 -header-include-format=json -header-include-filtering=only-direct-system -header-include-file %t.txt -emit-module -x c -fmodules -fmodule-name=X %s -o /dev/null
+module X {}

clang/test/Preprocessor/print-header-json.c

@@ -1,20 +1,33 @@
 // RUN: %clang_cc1 -E -header-include-format=json -header-include-filtering=only-direct-system -header-include-file %t.txt -I %S/Inputs/print-header-json -isystem %S/Inputs/print-header-json/system %s
 // RUN: cat %t.txt | FileCheck %s --check-prefix=SUPPORTED
+
 // RUN: not %clang_cc1 -E -header-include-format=textual -header-include-filtering=only-direct-system -header-include-file %t.txt -I %S/Inputs/print-header-json -isystem %S/Inputs/print-header-json/system %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=UNSUPPORTED0
 // RUN: not %clang_cc1 -E -header-include-format=json -header-include-filtering=none -header-include-file %t.txt -I %S/Inputs/print-header-json -isystem %S/Inputs/print-header-json/system %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=UNSUPPORTED1
-// RUN: rm %t.txt
-// RUN: env CC_PRINT_HEADERS_FORMAT=json CC_PRINT_HEADERS_FILTERING=only-direct-system CC_PRINT_HEADERS_FILE=%t.txt %clang -fsyntax-only -I %S/Inputs/print-header-json -isystem %S/Inputs/print-header-json/system %s -o /dev/null
 // RUN: env CC_PRINT_HEADERS_FORMAT=textual CC_PRINT_HEADERS_FILTERING=only-direct-system CC_PRINT_HEADERS_FILE=%t.txt not %clang -fsyntax-only -I %S/Inputs/print-header-json -isystem %S/Inputs/print-header-json/system %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=UNSUPPORTED2
 // RUN: env CC_PRINT_HEADERS_FORMAT=json CC_PRINT_HEADERS_FILTERING=none CC_PRINT_HEADERS_FILE=%t.txt not %clang -fsyntax-only -I %S/Inputs/print-header-json -isystem %S/Inputs/print-header-json/system %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=UNSUPPORTED3
+// RUN: env CC_PRINT_HEADERS_FORMAT=json CC_PRINT_HEADERS_FILE=%t.txt not %clang -fsyntax-only -I %S/Inputs/print-header-json -isystem %S/Inputs/print-header-json/system %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=UNSUPPORTED4
+// RUN: not %clang_cc1 -E -header-include-filtering=only-direct-system -header-include-file %t.txt -I %S/Inputs/print-header-json -isystem %S/Inputs/print-header-json/system %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=UNSUPPORTED5
+// RUN: not %clang_cc1 -E -header-include-format=json -header-include-file %t.txt -I %S/Inputs/print-header-json -isystem %S/Inputs/print-header-json/system %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=UNSUPPORTED6
+
+// RUN: rm %t.txt
+// RUN: env CC_PRINT_HEADERS_FORMAT=json CC_PRINT_HEADERS_FILTERING=only-direct-system CC_PRINT_HEADERS_FILE=%t.txt %clang -fsyntax-only -I %S/Inputs/print-header-json -isystem %S/Inputs/print-header-json/system %s -o /dev/null
 // RUN: cat %t.txt | FileCheck %s --check-prefix=SUPPORTED
 
+// RUN: rm %t.txt
+// RUN: env CC_PRINT_HEADERS_FORMAT=json CC_PRINT_HEADERS_FILTERING=direct-per-file CC_PRINT_HEADERS_FILE=%t.txt %clang -fsyntax-only -I %S/Inputs/print-header-json -isystem %S/Inputs/print-header-json/system %s -o /dev/null
+// RUN: cat %t.txt | FileCheck %s --check-prefix=SUPPORTED_PERFILE
+
 #include "system0.h"
 #include "header0.h"
 #include "system2.h"
 
 // SUPPORTED: {"source":"{{[^,]*}}print-header-json.c","includes":["{{[^,]*}}system0.h","{{[^,]*}}system3.h","{{[^,]*}}system2.h"]}
+// SUPPORTED_PERFILE: [{"source":"{{[^,]*}}print-header-json.c","includes":["{{[^,]*}}system0.h","{{[^,]*}}header0.h","{{[^,]*}}system2.h"]},{"source":"{{[^,]*}}header0.h","includes":["{{[^,]*}}system3.h","{{[^,]*}}header1.h","{{[^,]*}}header2.h"]}]
 
 // UNSUPPORTED0: error: unsupported combination: -header-include-format=textual and -header-include-filtering=only-direct-system
 // UNSUPPORTED1: error: unsupported combination: -header-include-format=json and -header-include-filtering=none
 // UNSUPPORTED2: error: unsupported combination: CC_PRINT_HEADERS_FORMAT=textual and CC_PRINT_HEADERS_FILTERING=only-direct-system
 // UNSUPPORTED3: error: unsupported combination: CC_PRINT_HEADERS_FORMAT=json and CC_PRINT_HEADERS_FILTERING=none
+// UNSUPPORTED4: error: environment variable CC_PRINT_HEADERS_FORMAT=json requires a compatible value for CC_PRINT_HEADERS_FILTERING
+// UNSUPPORTED5: error: -header-include-filtering=only-direct-system requires a compatible value for -header-include-format
+// UNSUPPORTED6: error: -header-include-format=json requires a compatible value for -header-include-filtering

clang/tools/driver/driver.cpp

@@ -167,6 +167,11 @@ static bool SetBackdoorDriverOutputsFromEnvVars(Driver &TheDriver) {
       }
 
       const char *FilteringStr = ::getenv("CC_PRINT_HEADERS_FILTERING");
+      if (!FilteringStr) {
+        TheDriver.Diag(clang::diag::err_drv_print_header_env_var_invalid_format)
+            << EnvVar;
+        return false;
+      }
       HeaderIncludeFilteringKind Filtering;
       if (!stringToHeaderIncludeFiltering(FilteringStr, Filtering)) {
         TheDriver.Diag(clang::diag::err_drv_print_header_env_var)
@@ -177,7 +182,7 @@ static bool SetBackdoorDriverOutputsFromEnvVars(Driver &TheDriver) {
       if ((TheDriver.CCPrintHeadersFormat == HIFMT_Textual &&
            Filtering != HIFIL_None) ||
           (TheDriver.CCPrintHeadersFormat == HIFMT_JSON &&
-           Filtering != HIFIL_Only_Direct_System)) {
+           Filtering == HIFIL_None)) {
         TheDriver.Diag(clang::diag::err_drv_print_header_env_var_combination)
             << EnvVar << FilteringStr;
         return false;