[clang][dep-scan] Resolve lexer crash from a permutation of invalid tokens

Sometimes when a user writes invalid code, the minimization used for
scanning can create a stream of tokens that is invalid at lex time.
This patch protects against the case where theres valid import directives
discovered in the middle of an invalid `import` declaration.

resolves: rdar://152335844

Author: cyndyishida

clang/include/clang/Basic/DiagnosticLexKinds.td

@@ -1029,7 +1029,8 @@ def err_dep_source_scanner_missing_semi_after_at_import : Error<
   "could not find ';' after @import">;
 def err_dep_source_scanner_unexpected_tokens_at_import : Error<
   "unexpected extra tokens at end of @import declaration">;
-
+def err_dep_source_scanner_unexpected_tokens_in_directive_body
+    : Error<"unexpected extra tokens inside declaration">;
 }
 
 def err_pp_double_begin_pragma_unsafe_buffer_usage :

clang/lib/Lex/DependencyDirectivesScanner.cpp

@@ -503,6 +503,10 @@ bool Scanner::lexModuleDirectiveBody(DirectiveKind Kind, const char *&First,
           diag::err_dep_source_scanner_missing_semi_after_at_import);
     if (Tok.is(tok::semi))
       break;
+    if (Tok.is(tok::hash) || Tok.is(tok::at))
+      return reportError(
+          First,
+          diag::err_dep_source_scanner_unexpected_tokens_in_directive_body);
   }
   pushDirective(Kind);
   skipWhitespace(First, End);
@@ -846,6 +850,7 @@ bool Scanner::lexPPLine(const char *&First, const char *const End) {
   if (*First == '@')
     return lexAt(First, End);
 
+  // Handle module directives for C++20 modules.
   if (*First == 'i' || *First == 'e' || *First == 'm')
     return lexModule(First, End);
 

clang/test/Lexer/minimize_source_to_dependency_directives_invalid_import.m

@@ -0,0 +1,6 @@
+// RUN: %clang_cc1 -verify -print-dependency-directives-minimized-source %s 2>&1
+
+import <invalid.h>
+#import "invalid.h" // expected-error {{unexpected extra tokens inside declaration}}
+@;
+#pragma clang module import invalid

clang/unittests/Lex/DependencyDirectivesScannerTest.cpp

@@ -33,7 +33,7 @@ static bool minimizeSourceToDependencyDirectives(
 
   return false;
 }
-
+// Returns false on successful minimization.
 static bool minimizeSourceToDependencyDirectives(StringRef Input,
                                                  SmallVectorImpl<char> &Out) {
   SmallVector<dependency_directives_scan::Token, 16> Tokens;
@@ -677,13 +677,22 @@ TEST(MinimizeSourceToDependencyDirectivesTest, EmptyIncludesAndImports) {
                Out.data());
 }
 
-TEST(MinimizeSourceToDependencyDirectivesTest, AtImportFailures) {
+TEST(MinimizeSourceToDependencyDirectivesTest, ImportFailures) {
   SmallVector<char, 128> Out;
 
-  ASSERT_TRUE(minimizeSourceToDependencyDirectives("@import A\n", Out));
+  // Minimization can recover, and let the real compilation fail.
   ASSERT_FALSE(
       minimizeSourceToDependencyDirectives("@import MACRO(A);\n", Out));
   ASSERT_FALSE(minimizeSourceToDependencyDirectives("@import \" \";\n", Out));
+
+  ASSERT_FALSE(minimizeSourceToDependencyDirectives("import\n", Out));
+  EXPECT_STREQ("<TokBeforeEOF>\n", Out.data());
+
+  // Minimization cannot recover.
+  ASSERT_TRUE(minimizeSourceToDependencyDirectives("@import A\n", Out));
+  ASSERT_TRUE(minimizeSourceToDependencyDirectives("import <Foo.h>\n"
+                                                   "#include \"Foo.h\"",
+                                                   Out));
 }
 
 TEST(MinimizeSourceToDependencyDirectivesTest, RawStringLiteral) {