[flang] Rework preprocessor fix for replacement in kind suffixes

[klausler]

Recent work to better handle macro replacement in literal constant kind suffixes isn't handling fixed form well, leading to a crash in Fujitsu test 0113/0113_0073.F. The look-ahead needs to be done with the higher-level prescanner functions that skip over fixed form comment fields after column 72. Rework.

[llvmbot]

@llvm/pr-subscribers-flang-parser

Author: Peter Klausler (klausler)

Changes

Recent work to better handle macro replacement in literal constant kind suffixes isn't handling fixed form well, leading to a crash in Fujitsu test 0113/0113_0073.F. The look-ahead needs to be done with the higher-level prescanner functions that skip over fixed form comment fields after column 72. Rework.

---

Full diff: https://github.com/llvm/llvm-project/pull/135406.diff

2 Files Affected:

• (modified) flang/lib/Parser/prescan.cpp (+38-32)
• (added) flang/test/Preprocessing/bug518.F (+5)

diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index 43a6d8c76f067..52c0183a5f0f9 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -891,51 +891,57 @@ bool Prescanner::HandleExponent(TokenSequence &tokens) {
   if (char ed{ToLowerCaseLetter(*at_)}; ed == 'e' || ed == 'd') {
     // Do some look-ahead to ensure that this 'e'/'d' is an exponent,
     // not the start of an identifier that could be a macro.
-    const char *p{SkipWhiteSpace(at_ + 1)};
-    if (*p == '+' || *p == '-') {
-      p = SkipWhiteSpace(p + 1);
-    }
-    if (IsDecimalDigit(*p)) { // it's an exponent
-      EmitCharAndAdvance(tokens, ed);
-      if (*at_ == '+' || *at_ == '-') {
-        EmitCharAndAdvance(tokens, *at_);
-      }
+    const char *startAt{at_};
+    int startColumn{column_};
+    TokenSequence possible;
+    EmitCharAndAdvance(possible, *at_);
+    if (*at_ == '+' || *at_ == '-') {
+      EmitCharAndAdvance(possible, *at_);
+    }
+    if (IsDecimalDigit(*at_)) { // it's an exponent; scan it
       while (IsDecimalDigit(*at_)) {
-        EmitCharAndAdvance(tokens, *at_);
+        EmitCharAndAdvance(possible, *at_);
       }
+      possible.CloseToken();
+      tokens.CloseToken();
+      tokens.Put(possible);
       return true;
     }
+    // Not an exponent; backtrack
+    at_ = startAt;
+    column_ = startColumn;
   }
   return false;
 }
 
 bool Prescanner::HandleKindSuffix(TokenSequence &tokens) {
-  if (*at_ == '_' && IsLegalInIdentifier(at_[1])) {
-    // The kind specifier might be a macro (with or without its leading
-    // underscore); put it into its own token if it has been defined.
-    const char *p{at_ + 1};
-    while (IsLegalInIdentifier(*++p)) {
-    }
-    if (CharBlock id{at_, static_cast<std::size_t>(p - at_)};
-        preprocessor_.IsNameDefined(id)) {
-      // In 1.0e0_foo, "_foo" is a defined name; retain the
-      // underscore
-      tokens.CloseToken();
-    } else {
-      EmitCharAndAdvance(tokens, '_');
-      if (CharBlock id{at_, static_cast<std::size_t>(p - at_)};
-          preprocessor_.IsNameDefined(id)) {
-        // In 1.0e0_foo, "foo" is a defined name
-        tokens.CloseToken();
-      }
-    }
+  if (*at_ != '_') {
+    return false;
+  }
+  TokenSequence withUnderscore, separate;
+  EmitChar(withUnderscore, '_');
+  EmitCharAndAdvance(separate, '_');
+  if (IsLegalInIdentifier(*at_)) {
+    separate.CloseToken();
+    EmitChar(withUnderscore, *at_);
+    EmitCharAndAdvance(separate, *at_);
     while (IsLegalInIdentifier(*at_)) {
-      EmitCharAndAdvance(tokens, *at_);
+      EmitChar(withUnderscore, *at_);
+      EmitCharAndAdvance(separate, *at_);
     }
-    return true;
+  }
+  withUnderscore.CloseToken();
+  separate.CloseToken();
+  tokens.CloseToken();
+  if (separate.SizeInTokens() == 2 &&
+      preprocessor_.IsNameDefined(separate.TokenAt(1)) &&
+      !preprocessor_.IsNameDefined(withUnderscore.ToCharBlock())) {
+    // "_foo" is not defined, but "foo" is
+    tokens.Put(separate); // '_' "foo"
   } else {
-    return false;
+    tokens.Put(withUnderscore); // "_foo"
   }
+  return true;
 }
 
 bool Prescanner::HandleExponentAndOrKindSuffix(TokenSequence &tokens) {
diff --git a/flang/test/Preprocessing/bug518.F b/flang/test/Preprocessing/bug518.F
new file mode 100644
index 0000000000000..346e04cc56d38
--- /dev/null
+++ b/flang/test/Preprocessing/bug518.F
@@ -0,0 +1,5 @@
+! RUN: %flang -fc1 -fdebug-unparse %s 2>&1 | FileCheck %s
+! CHECK: k=1_4
+                        k=                                            1_99999999
+     &4
+      end

[akuhlens]

LGTM