[SR-1545] Implement lexing of source conflict markers.

Lexing for conflict markers is inspired by the way clang handles them with a few
updates of our own.  Clang's lexer currently searches for the start of the
conflict marker, attempts to find the divider points, lexes that, then finds the
end.  We, unfortunately, cannot be so forgiving because of operator overloads.
Instead, we search for the start and end markers and ignore all text in between.
Even if what is found is not conflict markers, it certainly is not valid Swift either.

Author: CodaFi

include/swift/AST/DiagnosticsParse.def

@@ -141,7 +141,8 @@ ERROR(lex_editor_placeholder,none,
       "editor placeholder in source file", ())
 WARNING(lex_editor_placeholder_in_playground,none,
         "editor placeholder in source file", ())
-
+ERROR(lex_conflict_marker_in_file,none,
+      "source control conflict marker in source file", ())
 //------------------------------------------------------------------------------
 // Declaration parsing diagnostics
 //------------------------------------------------------------------------------

include/swift/Parse/Lexer.h

@@ -36,6 +36,16 @@ enum class CommentRetentionMode {
   ReturnAsTokens,
 };
 
+/// Kinds of conflict marker which the lexer might encounter.
+enum class ConflictMarkerKind {
+  /// A normal or diff3 conflict marker, initiated by at least 7 "<"s,
+  /// separated by at least 7 "="s or "|"s, and terminated by at least 7 ">"s.
+  Normal,
+  /// A Perforce-style conflict marker, initiated by 4 ">"s,
+  /// separated by 4 "="s, and terminated by 4 "<"s.
+  Perforce
+};
+  
 class Lexer {
   const LangOptions &LangOpts;
   const SourceManager &SourceMgr;
@@ -438,6 +448,10 @@ class Lexer {
 
   void tryLexEditorPlaceholder();
   const char *findEndOfCurlyQuoteStringLiteral(const char*);
+
+  /// Try to lex conflict markers by checking for the presence of the start and
+  /// end of the marker in diff3 or Perforce style respectively.
+  bool tryLexConflictMarker();
 };
 
 } // end namespace swift

lib/Parse/Lexer.cpp

@@ -1373,6 +1373,61 @@ void Lexer::lexEscapedIdentifier() {
   formToken(tok::backtick, Quote);
 }
 
+/// Find the end of a version control conflict marker.
+static const char *findConflictEnd(const char *CurPtr, const char *BufferEnd,
+                                   ConflictMarkerKind CMK) {
+  StringRef terminator = CMK == ConflictMarkerKind::Perforce ? "<<<<\n"
+                                                             : ">>>>>>> ";
+  size_t termLen = terminator.size();
+  
+  // Get a reference to the rest of the buffer minus the length of the start
+  // of the conflict marker.
+  auto restOfBuffer = StringRef(CurPtr, BufferEnd - CurPtr).substr(termLen);
+  size_t endPos = restOfBuffer.find(terminator);
+  while (endPos != StringRef::npos) {
+    // Must occur at start of line.
+    if (endPos != 0 &&
+        (restOfBuffer[endPos - 1] == '\r' || restOfBuffer[endPos - 1] == '\n'))
+    {
+      return restOfBuffer.data() + endPos;
+    }
+    restOfBuffer = restOfBuffer.substr(endPos + termLen);
+    endPos = restOfBuffer.find(terminator);
+  }
+  return nullptr;
+}
+
+bool Lexer::tryLexConflictMarker() {
+  const char *Ptr = CurPtr - 1;
+
+  // Only a conflict marker if it starts at the beginning of a line.
+  if (Ptr != BufferStart && Ptr[-1] != '\n' && Ptr[-1] != '\r')
+    return false;
+  
+  // Check to see if we have <<<<<<< or >>>>.
+  StringRef restOfBuffer(Ptr, BufferEnd - Ptr);
+  if (!restOfBuffer.startswith("<<<<<<< ") && !restOfBuffer.startswith(">>>> "))
+    return false;
+  
+  ConflictMarkerKind Kind = *Ptr == '<' ? ConflictMarkerKind::Normal
+                                        : ConflictMarkerKind::Perforce;
+  if (const char *End = findConflictEnd(Ptr, BufferEnd, Kind)) {
+    // Diagnose at the conflict marker, then jump ahead to the end.
+    diagnose(CurPtr, diag::lex_conflict_marker_in_file);
+    CurPtr = End;
+    
+    // Skip ahead to the end of the marker.
+    if (CurPtr != BufferEnd)
+      skipToEndOfLine();
+    
+    return true;
+  }
+  
+  // No end of conflict marker found.
+  return false;
+}
+
+
 void Lexer::tryLexEditorPlaceholder() {
   assert(CurPtr[-1] == '<' && CurPtr[0] == '#');
   const char *TokStart = CurPtr-1;
@@ -1689,10 +1744,17 @@ void Lexer::lexImpl() {
   case '<':
     if (CurPtr[0] == '#')
       return tryLexEditorPlaceholder();
-    SWIFT_FALLTHROUGH;
+    else if (CurPtr[0] == '<' && tryLexConflictMarker())
+      goto Restart;
+    return lexOperatorIdentifier();
 
-  case '=': case '-': case '+': case '*': case '>':
-  case '&': case '|': case '^': case '~': case '.':
+  case '>':
+    if (CurPtr[0] == '>' && tryLexConflictMarker())
+      goto Restart;
+    return lexOperatorIdentifier();
+ 
+  case '=': case '-': case '+': case '*':
+  case '&': case '|':  case '^': case '~': case '.':
     return lexOperatorIdentifier();
 
   case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G':

test/Parse/conflict_markers.swift

@@ -0,0 +1,88 @@
+// RUN: %target-parse-verify-swift
+
+// Conflict marker parsing should never conflict with operator parsing.
+
+prefix operator <<<<<<< {}
+infix operator <<<<<<< {}
+
+prefix func <<<<<<< (x : String) {}
+func <<<<<<< (x : String, y : String) {}
+
+prefix operator >>>>>>> {}
+infix operator >>>>>>> {}
+
+prefix func >>>>>>> (x : String) {}
+func >>>>>>> (x : String, y : String) {}
+
+// diff3-style conflict markers
+
+<<<<<<< HEAD:conflict_markers.swift // expected-error {{source control conflict marker in source file}}
+var a : String = "A"
+var b : String = "b"
+=======
+var a : String = "a"
+var b : String = "B"
+>>>>>>> 18844bc65229786b96b89a9fc7739c0fc897905e:conflict_markers.swift
+print(a + b) // expected-error {{use of unresolved identifier 'a'}} expected-error {{use of unresolved identifier 'b'}}
+
+<<<<<<< HEAD:conflict_markers.swift // expected-error {{source control conflict marker in source file}}
+======= 
+var d : String = "D"
+>>>>>>> 18844bc65229786b96b89a9fc7739c0fc897905e:conflict_markers.swift
+print(d) // expected-error {{use of unresolved identifier 'd'}}
+
+<<<<<<<"HEAD:fake_conflict_markers.swift" // No error
+>>>>>>>"18844bc65229786b96b89a9fc7739c0fc897905e:fake_conflict_markers.swift" // No error
+
+<<<<<<< HEAD:conflict_markers.swift // expected-error {{source control conflict marker in source file}}
+<<<<<<<"HEAD:fake_conflict_markers.swift"
+var fake_b : String = "a"
+>>>>>>>"18844bc65229786b96b89a9fc7739c0fc897905e:fake_conflict_markers.swift"
+=======
+<<<<<<<"HEAD:fake_conflict_markers.swift"
+var fake_c : String = "a"
+>>>>>>>"18844bc65229786b96b89a9fc7739c0fc897905e:fake_conflict_markers.swift"
+>>>>>>> 18844bc65229786b96b89a9fc7739c0fc897905e:conflict_markers.swift
+print(fake_b + fake_c) // expected-error {{use of unresolved identifier 'fake_b'}} expected-error {{use of unresolved identifier 'fake_c'}}
+
+// Disambiguating conflict markers from operator applications.
+
+_ = {
+  
+// Conflict marker.
+  
+let a = "a", b = "b"
+a // expected-warning {{expression of type 'String' is unused}}
+<<<<<<< b // expected-error {{source control conflict marker in source file}}
+a
+>>>>>>> b
+  
+// Not a conflict marker.
+  
+a
+  <<<<<<< b
+a
+  >>>>>>> b
+}()
+  
+// Perforce-style conflict markers
+
+>>>> ORIGINAL // expected-error {{source control conflict marker in source file}}
+var a : String = "A"
+var b : String = "B"
+==== THEIRS
+var a : String = "A"
+var b : String = "b"
+==== YOURS
+var a : String = "a"
+var b : String = "B"
+<<<<
+print(a + b) // expected-error {{use of unresolved identifier 'a'}} expected-error {{use of unresolved identifier 'b'}}
+
+>>>> ORIGINAL // expected-error {{source control conflict marker in source file}}
+==== THEIRS
+==== YOURS
+var d : String = "D"
+<<<<
+print(d) // expected-error {{use of unresolved identifier 'd'}}
+