Pass LangOpts from CompilerInstance to DependencyScanningWorker

This commit fixes #88896
by passing LangOpts from the CompilerInstance to
DependencyScanningWorker so that the original LangOpts are
preserved/respected. This makes for more accurate parsing/lexing when
certain language versions or features specific to versions are to be used.

Author: nishithshah2211

clang/include/clang/Lex/DependencyDirectivesScanner.h

@@ -17,6 +17,7 @@
 #ifndef LLVM_CLANG_LEX_DEPENDENCYDIRECTIVESSCANNER_H
 #define LLVM_CLANG_LEX_DEPENDENCYDIRECTIVESSCANNER_H
 
+#include "clang/Basic/LangOptions.h"
 #include "clang/Basic/SourceLocation.h"
 #include "llvm/ADT/ArrayRef.h"
 
@@ -117,7 +118,7 @@ struct Directive {
 bool scanSourceForDependencyDirectives(
     StringRef Input, SmallVectorImpl<dependency_directives_scan::Token> &Tokens,
     SmallVectorImpl<dependency_directives_scan::Directive> &Directives,
-    DiagnosticsEngine *Diags = nullptr,
+    const LangOptions &LangOpts, DiagnosticsEngine *Diags = nullptr,
     SourceLocation InputSourceLoc = SourceLocation());
 
 /// Print the previously scanned dependency directives as minimized source text.

clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h

@@ -363,7 +363,8 @@ class DependencyScanningWorkerFilesystem
   ///
   /// Returns true if the directive tokens are populated for this file entry,
   /// false if not (i.e. this entry is not a file or its scan fails).
-  bool ensureDirectiveTokensArePopulated(EntryRef Entry);
+  bool ensureDirectiveTokensArePopulated(EntryRef Entry,
+                                         const LangOptions &LangOpts);
 
   /// Check whether \p Path exists. By default checks cached result of \c
   /// status(), and falls back on FS if unable to do so.

clang/lib/Frontend/FrontendActions.cpp

@@ -1168,8 +1168,8 @@ void PrintDependencyDirectivesSourceMinimizerAction::ExecuteAction() {
   llvm::SmallVector<dependency_directives_scan::Token, 16> Tokens;
   llvm::SmallVector<dependency_directives_scan::Directive, 32> Directives;
   if (scanSourceForDependencyDirectives(
-          FromFile.getBuffer(), Tokens, Directives, &CI.getDiagnostics(),
-          SM.getLocForStartOfFile(SM.getMainFileID()))) {
+          FromFile.getBuffer(), Tokens, Directives, CI.getLangOpts(),
+          &CI.getDiagnostics(), SM.getLocForStartOfFile(SM.getMainFileID()))) {
     assert(CI.getDiagnostics().hasErrorOccurred() &&
            "no errors reported for failure");
 

clang/lib/Lex/DependencyDirectivesScanner.cpp

@@ -62,14 +62,17 @@ struct DirectiveWithTokens {
 struct Scanner {
   Scanner(StringRef Input,
           SmallVectorImpl<dependency_directives_scan::Token> &Tokens,
-          DiagnosticsEngine *Diags, SourceLocation InputSourceLoc)
+          DiagnosticsEngine *Diags, SourceLocation InputSourceLoc,
+          const LangOptions &LangOpts)
       : Input(Input), Tokens(Tokens), Diags(Diags),
-        InputSourceLoc(InputSourceLoc), LangOpts(getLangOptsForDepScanning()),
-        TheLexer(InputSourceLoc, LangOpts, Input.begin(), Input.begin(),
+        InputSourceLoc(InputSourceLoc),
+        LangOpts(getLangOptsForDepScanning(LangOpts)),
+        TheLexer(InputSourceLoc, this->LangOpts, Input.begin(), Input.begin(),
                  Input.end()) {}
 
-  static LangOptions getLangOptsForDepScanning() {
-    LangOptions LangOpts;
+  static LangOptions
+  getLangOptsForDepScanning(const LangOptions &invocationLangOpts) {
+    LangOptions LangOpts(invocationLangOpts);
     // Set the lexer to use 'tok::at' for '@', instead of 'tok::unknown'.
     LangOpts.ObjC = true;
     LangOpts.LineComment = true;
@@ -700,7 +703,7 @@ bool Scanner::lex_Pragma(const char *&First, const char *const End) {
   SmallVector<dependency_directives_scan::Token> DiscardTokens;
   const char *Begin = Buffer.c_str();
   Scanner PragmaScanner{StringRef(Begin, Buffer.size()), DiscardTokens, Diags,
-                        InputSourceLoc};
+                        InputSourceLoc, LangOptions()};
 
   PragmaScanner.TheLexer.setParsingPreprocessorDirective(true);
   if (PragmaScanner.lexPragma(Begin, Buffer.end()))
@@ -950,9 +953,10 @@ bool Scanner::scan(SmallVectorImpl<Directive> &Directives) {
 
 bool clang::scanSourceForDependencyDirectives(
     StringRef Input, SmallVectorImpl<dependency_directives_scan::Token> &Tokens,
-    SmallVectorImpl<Directive> &Directives, DiagnosticsEngine *Diags,
-    SourceLocation InputSourceLoc) {
-  return Scanner(Input, Tokens, Diags, InputSourceLoc).scan(Directives);
+    SmallVectorImpl<Directive> &Directives, const LangOptions &LangOpts,
+    DiagnosticsEngine *Diags, SourceLocation InputSourceLoc) {
+  return Scanner(Input, Tokens, Diags, InputSourceLoc, LangOpts)
+      .scan(Directives);
 }
 
 void clang::printDependencyDirectivesAsSource(

clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp

@@ -42,7 +42,7 @@ DependencyScanningWorkerFilesystem::readFile(StringRef Filename) {
 }
 
 bool DependencyScanningWorkerFilesystem::ensureDirectiveTokensArePopulated(
-    EntryRef Ref) {
+    EntryRef Ref, const LangOptions &LangOpts) {
   auto &Entry = Ref.Entry;
 
   if (Entry.isError() || Entry.isDirectory())
@@ -66,7 +66,7 @@ bool DependencyScanningWorkerFilesystem::ensureDirectiveTokensArePopulated(
   // dependencies.
   if (scanSourceForDependencyDirectives(Contents->Original->getBuffer(),
                                         Contents->DepDirectiveTokens,
-                                        Directives)) {
+                                        Directives, LangOpts)) {
     Contents->DepDirectiveTokens.clear();
     // FIXME: Propagate the diagnostic if desired by the client.
     Contents->DepDirectives.store(new std::optional<DependencyDirectivesTy>());

clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp

@@ -366,11 +366,12 @@ class DependencyScanningAction : public tooling::ToolAction {
     // Use the dependency scanning optimized file system if requested to do so.
     if (DepFS)
       ScanInstance.getPreprocessorOpts().DependencyDirectivesForFile =
-          [LocalDepFS = DepFS](FileEntryRef File)
+          [LocalDepFS = DepFS,
+           &LangOpts = ScanInstance.getLangOpts()](FileEntryRef File)
           -> std::optional<ArrayRef<dependency_directives_scan::Directive>> {
         if (llvm::ErrorOr<EntryRef> Entry =
                 LocalDepFS->getOrCreateFileSystemEntry(File.getName()))
-          if (LocalDepFS->ensureDirectiveTokensArePopulated(*Entry))
+          if (LocalDepFS->ensureDirectiveTokensArePopulated(*Entry, LangOpts))
             return Entry->getDirectiveTokens();
         return std::nullopt;
       };

clang/unittests/Lex/DependencyDirectivesScannerTest.cpp

@@ -17,11 +17,11 @@ using namespace clang::dependency_directives_scan;
 static bool minimizeSourceToDependencyDirectives(
     StringRef Input, SmallVectorImpl<char> &Out,
     SmallVectorImpl<dependency_directives_scan::Token> &Tokens,
-    SmallVectorImpl<Directive> &Directives) {
+    SmallVectorImpl<Directive> &Directives, const LangOptions &LangOpts) {
   Out.clear();
   Tokens.clear();
   Directives.clear();
-  if (scanSourceForDependencyDirectives(Input, Tokens, Directives))
+  if (scanSourceForDependencyDirectives(Input, Tokens, Directives, LangOpts))
     return true;
 
   raw_svector_ostream OS(Out);
@@ -38,7 +38,9 @@ static bool minimizeSourceToDependencyDirectives(StringRef Input,
                                                  SmallVectorImpl<char> &Out) {
   SmallVector<dependency_directives_scan::Token, 16> Tokens;
   SmallVector<Directive, 32> Directives;
-  return minimizeSourceToDependencyDirectives(Input, Out, Tokens, Directives);
+  LangOptions LangOpts;
+  return minimizeSourceToDependencyDirectives(Input, Out, Tokens, Directives,
+                                              LangOpts);
 }
 
 namespace {
@@ -47,16 +49,19 @@ TEST(MinimizeSourceToDependencyDirectivesTest, Empty) {
   SmallVector<char, 128> Out;
   SmallVector<dependency_directives_scan::Token, 4> Tokens;
   SmallVector<Directive, 4> Directives;
+  LangOptions LangOpts;
 
   ASSERT_FALSE(
-      minimizeSourceToDependencyDirectives("", Out, Tokens, Directives));
+      minimizeSourceToDependencyDirectives("", Out, Tokens, Directives,
+                                           LangOpts));
   EXPECT_TRUE(Out.empty());
   EXPECT_TRUE(Tokens.empty());
   ASSERT_EQ(1u, Directives.size());
   ASSERT_EQ(pp_eof, Directives.back().Kind);
 
   ASSERT_FALSE(minimizeSourceToDependencyDirectives("abc def\nxyz", Out, Tokens,
-                                                    Directives));
+                                                    Directives,
+                                                    LangOpts));
   EXPECT_STREQ("<TokBeforeEOF>\n", Out.data());
   EXPECT_TRUE(Tokens.empty());
   ASSERT_EQ(2u, Directives.size());
@@ -68,6 +73,7 @@ TEST(MinimizeSourceToDependencyDirectivesTest, AllTokens) {
   SmallVector<char, 128> Out;
   SmallVector<dependency_directives_scan::Token, 4> Tokens;
   SmallVector<Directive, 4> Directives;
+  LangOptions LangOpts;
 
   ASSERT_FALSE(
       minimizeSourceToDependencyDirectives("#define A\n"
@@ -92,7 +98,7 @@ TEST(MinimizeSourceToDependencyDirectivesTest, AllTokens) {
                                            "export module m;\n"
                                            "import m;\n"
                                            "#pragma clang system_header\n",
-                                           Out, Tokens, Directives));
+                                           Out, Tokens, Directives, LangOpts));
   EXPECT_EQ(pp_define, Directives[0].Kind);
   EXPECT_EQ(pp_undef, Directives[1].Kind);
   EXPECT_EQ(pp_endif, Directives[2].Kind);
@@ -145,9 +151,11 @@ TEST(MinimizeSourceToDependencyDirectivesTest, Define) {
   SmallVector<char, 128> Out;
   SmallVector<dependency_directives_scan::Token, 4> Tokens;
   SmallVector<Directive, 4> Directives;
+  LangOptions LangOpts;
 
   ASSERT_FALSE(minimizeSourceToDependencyDirectives("#define MACRO", Out,
-                                                    Tokens, Directives));
+                                                    Tokens, Directives,
+                                                    LangOpts));
   EXPECT_STREQ("#define MACRO\n", Out.data());
   ASSERT_EQ(4u, Tokens.size());
   ASSERT_EQ(2u, Directives.size());
@@ -838,6 +846,7 @@ TEST(MinimizeSourceToDependencyDirectivesTest, PragmaOnce) {
   SmallVector<char, 128> Out;
   SmallVector<dependency_directives_scan::Token, 4> Tokens;
   SmallVector<Directive, 4> Directives;
+  LangOptions LangOpts;
 
   StringRef Source = R"(// comment
 #pragma once
@@ -846,7 +855,8 @@ TEST(MinimizeSourceToDependencyDirectivesTest, PragmaOnce) {
 _Pragma("once")
 )";
   ASSERT_FALSE(
-      minimizeSourceToDependencyDirectives(Source, Out, Tokens, Directives));
+      minimizeSourceToDependencyDirectives(Source, Out, Tokens,
+                                           Directives, LangOpts));
   EXPECT_STREQ("#pragma once\n#include <test.h>\n_Pragma(\"once\")\n",
                Out.data());
   ASSERT_EQ(Directives.size(), 4u);
@@ -926,6 +936,7 @@ TEST(MinimizeSourceToDependencyDirectivesTest, CxxModules) {
   SmallVector<char, 128> Out;
   SmallVector<dependency_directives_scan::Token, 4> Tokens;
   SmallVector<Directive, 4> Directives;
+  LangOptions LangOpts;
 
   StringRef Source = R"(
     module;
@@ -955,7 +966,8 @@ ort \
     }
     )";
   ASSERT_FALSE(
-      minimizeSourceToDependencyDirectives(Source, Out, Tokens, Directives));
+      minimizeSourceToDependencyDirectives(Source, Out, Tokens,
+                                           Directives, LangOpts));
   EXPECT_STREQ("#include \"textual-header.h\"\nexport module m;"
                "exp\\\nort import:l[[rename]];"
                "import<<=3;import a b d e d e f e;"

clang/unittests/Lex/PPDependencyDirectivesTest.cpp

@@ -104,14 +104,15 @@ TEST_F(PPDependencyDirectivesTest, MacroGuard) {
     SmallVector<dependency_directives_scan::Directive> Directives;
   };
   SmallVector<std::unique_ptr<DepDirectives>> DepDirectivesObjects;
+  LangOptions LangOpts;
 
   auto getDependencyDirectives = [&](FileEntryRef File)
       -> std::optional<ArrayRef<dependency_directives_scan::Directive>> {
     DepDirectivesObjects.push_back(std::make_unique<DepDirectives>());
     StringRef Input = (*FileMgr.getBufferForFile(File))->getBuffer();
     bool Err = scanSourceForDependencyDirectives(
         Input, DepDirectivesObjects.back()->Tokens,
-        DepDirectivesObjects.back()->Directives);
+        DepDirectivesObjects.back()->Directives, LangOpts);
     EXPECT_FALSE(Err);
     return llvm::ArrayRef(DepDirectivesObjects.back()->Directives);
   };