[Syntax] add UTF-8 BOM support to libSyntax

Author: omochi

lib/Parse/Lexer.cpp

@@ -188,9 +188,9 @@ Lexer::Lexer(const LangOptions &Options,
     .StartsWith("\xEF\xBB\xBF", 3)
     .Default(0);
 
-  // Since the UTF-8 BOM doesn't carry information (UTF-8 has no dependency
-  // on byte order), throw it away.
-  CurPtr = BufferStart + BOMLength;
+  // Keep information about existance of UTF-8 BOM for transparency source code
+  // editing with libSyntax.
+  CurPtr = BufferStart;
   ContentStart = BufferStart + BOMLength;
 
   // Initialize code completion.
@@ -2036,7 +2036,20 @@ void Lexer::lexImpl() {
     LeadingTrivia.clear();
     TrailingTrivia.clear();
   }
-  NextToken.setAtStartOfLine(CurPtr == ContentStart);
+  if (CurPtr == BufferStart) {
+    if (BufferStart < ContentStart) {
+      size_t BOMLen = ContentStart - BufferStart;
+      assert(BOMLen == 3 && "UTF-8 BOM is 3 bytes");
+      if (TriviaRetention == TriviaRetentionMode::WithTrivia) {
+        // Add UTF-8 BOM to LeadingTrivia.
+        LeadingTrivia.push_back(TriviaPiece::garbageText({CurPtr, BOMLen}));
+      }
+      CurPtr += BOMLen;
+    }
+    NextToken.setAtStartOfLine(true);
+  } else {
+    NextToken.setAtStartOfLine(false);
+  }
 
   // Remember where we started so that we can find the comment range.
   LastCommentBlockStart = CurPtr;

unittests/Parse/LexerTests.cpp

@@ -400,6 +400,7 @@ TEST_F(LexerTest, BOMNoCommentTrivia) {
   ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 14), Tok.getCommentRange().getStart());
   ASSERT_EQ(0u, Tok.getCommentRange().getByteLength());
   ASSERT_EQ((syntax::Trivia{{
+    syntax::TriviaPiece::garbageText("\xEF\xBB\xBF"),
     syntax::TriviaPiece::lineComment("// comment"),
     syntax::TriviaPiece::newlines(1)
   }}), LeadingTrivia);
@@ -440,6 +441,7 @@ TEST_F(LexerTest, BOMAttachCommentTrivia) {
   ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 3), Tok.getCommentRange().getStart());
   ASSERT_EQ(10u, Tok.getCommentRange().getByteLength());
   ASSERT_EQ((syntax::Trivia{{
+    syntax::TriviaPiece::garbageText("\xEF\xBB\xBF"),
     syntax::TriviaPiece::lineComment("// comment"),
     syntax::TriviaPiece::newlines(1)
   }}), LeadingTrivia);

unittests/Parse/LexerTriviaTests.cpp

@@ -102,9 +102,9 @@ TEST_F(LexerTriviaTest, TriviaHashbangAfterBOM) {
   ASSERT_EQ("aaa", Tok.getText());
   ASSERT_TRUE(Tok.isAtStartOfLine());
 
-  // FIXME: This should include UTF8-BOM as a GarbargeText trivia.
   ASSERT_EQ(LeadingTrivia,
-            (Trivia{{TriviaPiece::garbageText("#!/bin/swift"),
+            (Trivia{{TriviaPiece::garbageText("\xEF\xBB\xBF"),
+                     TriviaPiece::garbageText("#!/bin/swift"),
                      TriviaPiece::newlines(1)}}));
 }