UPSTREAM: [clang-repl] Fix printing preprocessed tokens and macros (llvm#104964)

(cherry picked from commit 947b9f5)

Author: hahnjo

clang/lib/Frontend/PrintPreprocessedOutput.cpp

@@ -833,8 +833,7 @@ static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok,
       PP.Lex(Tok);
       continue;
     } else if (Tok.is(tok::annot_repl_input_end)) {
-      PP.Lex(Tok);
-      continue;
+      // Fall through to exit the loop.
     } else if (Tok.is(tok::eod)) {
       // Don't print end of directive tokens, since they are typically newlines
       // that mess up our line tracking. These come from unknown pre-processor
@@ -922,7 +921,8 @@ static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok,
     Callbacks->setEmittedTokensOnThisLine();
     IsStartOfLine = false;
 
-    if (Tok.is(tok::eof)) break;
+    if (Tok.is(tok::eof) || Tok.is(tok::annot_repl_input_end))
+      break;
 
     PP.Lex(Tok);
   }
@@ -941,9 +941,7 @@ static void DoPrintMacros(Preprocessor &PP, raw_ostream *OS) {
   // the macro table at the end.
   PP.EnterMainSourceFile();
 
-  Token Tok;
-  do PP.Lex(Tok);
-  while (Tok.isNot(tok::eof));
+  PP.LexTokensUntilEOF();
 
   SmallVector<id_macro_pair, 128> MacrosByID;
   for (Preprocessor::macro_iterator I = PP.macro_begin(), E = PP.macro_end();

clang/test/Interpreter/preprocessor.cpp

@@ -0,0 +1,4 @@
+// RUN: %clang_cc1 -fincremental-extensions -E %s
+// RUN: %clang_cc1 -fincremental-extensions -E -dD %s
+// RUN: %clang_cc1 -fincremental-extensions -E -dI %s
+// RUN: %clang_cc1 -fincremental-extensions -E -dM %s