Merge pull request #5410 from rintaro/lex-hoist-se0037

[Lexer] Trim comments off operator-identifier before tokenize

Author: slavapestov

lib/Parse/Lexer.cpp

@@ -697,6 +697,17 @@ void Lexer::lexOperatorIdentifier() {
       break;
   } while (advanceIfValidContinuationOfOperator(CurPtr, BufferEnd));
 
+  if (CurPtr-TokStart > 2) {
+    // If there is a "//" or "/*" in the middle of an identifier token, 
+    // it starts a comment.
+    for (auto Ptr = TokStart+1; Ptr != CurPtr-1; ++Ptr) {
+      if (Ptr[0] == '/' && (Ptr[1] == '/' || Ptr[1] == '*')) {
+        CurPtr = Ptr;
+        break;
+      }
+    }
+  }
+
   // Decide between the binary, prefix, and postfix cases.
   // It's binary if either both sides are bound or both sides are not bound.
   // Otherwise, it's postfix if left-bound and prefix if right-bound.
@@ -771,27 +782,9 @@ void Lexer::lexOperatorIdentifier() {
       return formToken(tok::unknown, TokStart);
     }
   } else {
-    // If there is a "//" in the middle of an identifier token, it starts
-    // a single-line comment.
-    auto Pos = StringRef(TokStart, CurPtr-TokStart).find("//");
-    if (Pos != StringRef::npos) {
-      CurPtr = TokStart+Pos;
-      // Next token is a comment, which counts as whitespace.
-      rightBound = false;
-    }
-
-    // If there is a "/*" in the middle of an identifier token, it starts
-    // a multi-line comment.
-    Pos = StringRef(TokStart, CurPtr-TokStart).find("/*");
-    if (Pos != StringRef::npos) {
-      CurPtr = TokStart+Pos;
-      // Next token is a comment, which counts as whitespace.
-      rightBound = false;
-    }
-
     // Verify there is no "*/" in the middle of the identifier token, we reject
     // it as potentially ending a block comment.
-    Pos = StringRef(TokStart, CurPtr-TokStart).find("*/");
+    auto Pos = StringRef(TokStart, CurPtr-TokStart).find("*/");
     if (Pos != StringRef::npos) {
       diagnose(TokStart+Pos, diag::lex_unexpected_block_comment_end);
       return formToken(tok::unknown, TokStart);

test/Parse/comment_operator.swift

@@ -27,3 +27,9 @@ func test6() { _ = 1+/* */2 }                 // expected-error {{'+' is not a p
 _ = foo!// this is dangerous
 _ = 1 +/**/ 2
 _ = 1 +/* hi */2
+
+// Ensure built-in operators are properly tokenized.
+_ =/**/2
+_/**/= 2
+typealias A = () ->/* */()
+func test7(x: Int) { _ = x./* desc */ } // expected-error {{expected member name following '.'}}