[WIP] Implement SE-0039 (Modernizing Playground Literals) (#2215)

* Implement the majority of parsing support for SE-0039.

* Parse old object literals names using new syntax and provide FixIt.

For example, parse "#Image(imageLiteral:...)" and provide a FixIt to
change it to "#imageLiteral(resourceName:...)".  Now we see something like:

test.swift:4:9: error: '#Image' has been renamed to '#imageLiteral
var y = #Image(imageLiteral: "image.jpg")
        ^~~~~~ ~~~~~~~~~~~~
        #imageLiteral resourceName

Handling the old syntax, and providing a FixIt for that, will be handled in a separate
commit.

Needs tests.  Will be provided in later commit once full parsing support is done.

* Add back pieces of syntax map for object literals.

* Add parsing support for old object literal syntax.

... and provide fixits to new syntax.

Full tests to come in later commit.

* Improve parsing of invalid object literals with old syntax.

* Do not include bracket in code completion results.

* Remove defunct code in SyntaxModel.

* Add tests for migration fixits.

* Add literals to code completion overload tests.

@akyrtzi told me this should be fine.

* Clean up response tests not to include full paths.

* Further adjust offsets.

* Mark initializer for _ColorLiteralConvertible in UIKit as @nonobjc.

* Put attribute in the correct place.

Author: tkremenek

.gitignore

@@ -39,3 +39,4 @@ docs/_build
 #==============================================================================#
 CMakeCache.txt
 CMakeFiles
+.atom-build.json

include/swift/AST/DiagnosticsParse.def

@@ -1046,12 +1046,14 @@ ERROR(expected_rsquare_array_expr,PointsToFirstBadToken,
       "expected ']' in container literal expression", ())
 
 // Object literal expressions
-ERROR(expected_identifier_after_l_square_lit,none,
-      "expected identifier after '[#' in object literal expression", ())
+ERROR(expected_object_literal_identifier,none,
+      "expected identifier after '#' in object literal expression", ())
 ERROR(expected_arg_list_in_object_literal,none,
       "expected argument list in object literal", ())
-ERROR(expected_r_square_lit_after_object_literal,none,
-      "expected '#]' at end of object literal expression", ())
+ERROR(object_literal_legacy_name,none,
+      "'%0' has been renamed to '%1", (StringRef, StringRef))
+ERROR(legacy_object_literal_syntax,none,
+       "object literal syntax no longer uses '[# ... #]'", ())
 
 // If expressions
 ERROR(expected_expr_after_if_question,none,

include/swift/AST/DiagnosticsSema.def

@@ -514,8 +514,8 @@ ERROR(pattern_used_in_type,none,
 NOTE(note_module_as_type,none,
      "cannot use module %0 as a type", (Identifier))
 
-ERROR(use_unknown_object_literal,none,
-     "use of unknown object literal name %0", (Identifier))
+ERROR(use_unknown_object_literal_protocol,none,
+     "cannot deduce protocol for %0 literal", (StringRef))
 ERROR(object_literal_default_type_missing,none,
      "could not infer type of %0 literal", (StringRef))
 NOTE(object_literal_resolve_import,none,

include/swift/AST/Expr.h

@@ -831,36 +831,47 @@ class MagicIdentifierLiteralExpr : public LiteralExpr {
 };
 
 // ObjectLiteralExpr - An expression of the form
-// '[#Color(red: 1, blue: 0, green: 0, alpha: 1)#]' with a name and a list
+// '#colorLiteral(red: 1, blue: 0, green: 0, alpha: 1)' with a name and a list
 // argument. The components of the list argument are meant to be themselves
 // constant.
 class ObjectLiteralExpr : public LiteralExpr {
-  Identifier Name;
+public:
+  /// The kind of object literal.
+  enum LiteralKind : unsigned {
+#define POUND_OBJECT_LITERAL(Name, Desc, Proto) Name,
+#include "swift/Parse/Tokens.def"    
+  };
+
+private:
+  LiteralKind LitKind;
   Expr *Arg;
   Expr *SemanticExpr;
-
-  SourceLoc LLitLoc;
-  SourceLoc NameLoc;
-  SourceLoc RLitLoc;
+  SourceLoc PoundLoc;
 
 public:
-  ObjectLiteralExpr(SourceLoc LLitLoc, Identifier Name, SourceLoc NameLoc,
-                    Expr *Arg, SourceLoc RLitLoc, bool implicit = false)
+  ObjectLiteralExpr(SourceLoc PoundLoc, LiteralKind LitKind,
+                    Expr *Arg, bool implicit = false)
     : LiteralExpr(ExprKind::ObjectLiteral, implicit), 
-      Name(Name), Arg(Arg), SemanticExpr(nullptr),
-      LLitLoc(LLitLoc), NameLoc(NameLoc), RLitLoc(RLitLoc) {}
+      LitKind(LitKind), Arg(Arg), SemanticExpr(nullptr),
+      PoundLoc(PoundLoc) {}
 
-  Identifier getName() const { return Name; }
-  SourceLoc getNameLoc() const { return NameLoc; }
+  LiteralKind getLiteralKind() const { return LitKind; }
 
   Expr *getArg() const { return Arg; }
   void setArg(Expr *arg) { Arg = arg; }
 
   Expr *getSemanticExpr() const { return SemanticExpr; }
   void setSemanticExpr(Expr *expr) { SemanticExpr = expr; }
 
-  SourceLoc getSourceLoc() const { return NameLoc; }
-  SourceRange getSourceRange() const { return SourceRange(LLitLoc, RLitLoc); }
+  SourceLoc getSourceLoc() const { return PoundLoc; }
+  SourceRange getSourceRange() const { 
+    return SourceRange(PoundLoc, Arg->getEndLoc());
+  }
+
+  /// Return the string form of the literal name.
+  StringRef getLiteralKindRawName() const;
+
+  StringRef getLiteralKindPlainName() const;
 
   static bool classof(const Expr *E) {
     return E->getKind() == ExprKind::ObjectLiteral;

include/swift/Parse/Parser.h

@@ -1177,8 +1177,17 @@ class Parser {
 
   Expr *parseExprAnonClosureArg();
   ParserResult<Expr> parseExprList(tok LeftTok, tok RightTok);
+
+  // NOTE: used only for legacy support for old object literal syntax.
+  // Will be removed in the future.
   bool isCollectionLiteralStartingWithLSquareLit();
-  ParserResult<Expr> parseExprObjectLiteral();
+
+  /// Parse an object literal.
+  ///
+  /// \param LK The literal kind as determined by the first token.
+  /// \param NewName New name for a legacy literal.
+  ParserResult<Expr> parseExprObjectLiteral(ObjectLiteralExpr::LiteralKind LK,
+                                            StringRef NewName = StringRef());
   ParserResult<Expr> parseExprCallSuffix(ParserResult<Expr> fn,
                                          Identifier firstSelectorPiece
                                            = Identifier(),

include/swift/Parse/Tokens.def

@@ -38,6 +38,29 @@
 #define POUND_KEYWORD(kw)
 #endif
 
+/// POUND_NORMAL_KEYWORD(kw)
+///   Keywords except object and config keywords in the #foo namespace.
+#ifndef POUND_NORMAL_KEYWORD
+#define POUND_NORMAL_KEYWORD(kw) POUND_KEYWORD(kw)
+#endif
+
+/// POUND_OBJECT_LITERAL(kw, desc, proto)
+///   Every keyword in the #foo namespace representing an object literal.
+#ifndef POUND_OBJECT_LITERAL
+#define POUND_OBJECT_LITERAL(kw, desc, proto) POUND_KEYWORD(kw)
+#endif
+
+/// POUND_OLD_OBJECT_LITERAL(kw, new_kw, old_arg, new_arg)
+///   Every keyword in the #foo namespace representing an object literal.
+#ifndef POUND_OLD_OBJECT_LITERAL
+#define POUND_OLD_OBJECT_LITERAL(kw, new_kw, old_arg, new_arg) POUND_KEYWORD(kw)
+#endif
+
+/// POUND_CONFIG(kw)
+///   Every keyword in the #foo namespace representing a configuration.
+#ifndef POUND_CONFIG
+#define POUND_CONFIG(kw) POUND_KEYWORD(kw)
+#endif
 
 /// SIL_KEYWORD(kw)
 ///   Expands for every SIL keyword. These are only keywords when parsing SIL.
@@ -140,9 +163,6 @@ PUNCTUATOR(r_brace,       "}")
 PUNCTUATOR(l_square,      "[")
 PUNCTUATOR(r_square,      "]")
 
-PUNCTUATOR(l_square_lit,  "[#")
-PUNCTUATOR(r_square_lit,  "#]")
-
 PUNCTUATOR(period,        ".")
 PUNCTUATOR(period_prefix, ".")
 PUNCTUATOR(comma,         ",")
@@ -165,6 +185,11 @@ PUNCTUATOR(question_infix,"?")    // if not left-bound
 PUNCTUATOR(sil_dollar,    "$")    // Only in SIL mode.
 PUNCTUATOR(sil_exclamation, "!")    // Only in SIL mode.
 
+// Legacy punctuators used for migrating old object literal syntax.
+// NOTE: Remove in the future.
+PUNCTUATOR(l_square_lit,  "[#")
+PUNCTUATOR(r_square_lit,  "#]")
+
 // Keywords in the # namespace.  "if" becomes "tok::pound_if".
 POUND_KEYWORD(if)
 POUND_KEYWORD(else)
@@ -173,9 +198,22 @@ POUND_KEYWORD(endif)
 POUND_KEYWORD(line)
 POUND_KEYWORD(setline)
 POUND_KEYWORD(sourceLocation)
-POUND_KEYWORD(available)
 POUND_KEYWORD(selector)
 
+// Keywords in the # namespace that are build configurations.
+POUND_CONFIG(available)
+
+
+// Declaratively define object literals, including their
+// corresponding protocols.
+POUND_OBJECT_LITERAL(fileLiteral, "file reference", FileReferenceLiteralConvertible)
+POUND_OBJECT_LITERAL(imageLiteral, "image", ImageLiteralConvertible)
+POUND_OBJECT_LITERAL(colorLiteral, "color", ColorLiteralConvertible)
+
+POUND_OLD_OBJECT_LITERAL(FileReference, fileLiteral, fileReferenceLiteral, resourceName)
+POUND_OLD_OBJECT_LITERAL(Image, imageLiteral, imageLiteral, resourceName)
+POUND_OLD_OBJECT_LITERAL(Color, colorLiteral, colorLiteralRed, red)
+
 POUND_KEYWORD(file)
 POUND_KEYWORD(column)
 POUND_KEYWORD(function)
@@ -188,3 +226,7 @@ POUND_KEYWORD(dsohandle)
 #undef SIL_KEYWORD
 #undef PUNCTUATOR
 #undef POUND_KEYWORD
+#undef POUND_NORMAL_KEYWORD
+#undef POUND_OBJECT_LITERAL
+#undef POUND_OLD_OBJECT_LITERAL
+#undef POUND_CONFIG

lib/AST/ASTDumper.cpp

@@ -1623,9 +1623,8 @@ class PrintExpr : public ExprVisitor<PrintExpr> {
   }
 
   void visitObjectLiteralExpr(ObjectLiteralExpr *E) {
-    printCommon(E, "object_literal")
-      << " name=" << E->getName();
-    OS << '\n';
+    printCommon(E, "object_literal") 
+      << " kind='" << E->getLiteralKindPlainName() << "'\n";
     printRec(E->getArg());
   }
 

lib/AST/Expr.cpp

@@ -743,9 +743,8 @@ shallowCloneImpl(const MagicIdentifierLiteralExpr *E, ASTContext &Ctx) {
 
 static LiteralExpr *
 shallowCloneImpl(const ObjectLiteralExpr *E, ASTContext &Ctx) {
-  auto res = new (Ctx) ObjectLiteralExpr(E->getStartLoc(), E->getName(),
-                                         E->getNameLoc(), E->getArg(),
-                                         E->getEndLoc());
+  auto res = new (Ctx) ObjectLiteralExpr(E->getStartLoc(), E->getLiteralKind(),
+                                         E->getArg());
   res->setSemanticExpr(E->getSemanticExpr());
   return res;
 }
@@ -846,6 +845,22 @@ StringLiteralExpr::StringLiteralExpr(StringRef Val, SourceRange Range,
       unicode::isSingleExtendedGraphemeCluster(Val);
 }
 
+StringRef ObjectLiteralExpr::getLiteralKindRawName() const {
+  switch (LitKind) {
+#define POUND_OBJECT_LITERAL(Name, Desc, Proto) case Name: return #Name;
+#include "swift/Parse/Tokens.def"    
+  }
+  llvm_unreachable("unspecified literal");
+}
+
+StringRef ObjectLiteralExpr::getLiteralKindPlainName() const {
+  switch (LitKind) {
+#define POUND_OBJECT_LITERAL(Name, Desc, Proto) case Name: return Desc;
+#include "swift/Parse/Tokens.def"    
+  }
+  llvm_unreachable("unspecified literal");
+}
+
 void DeclRefExpr::setSpecialized() {
   if (isSpecialized())
     return;

lib/IDE/CodeCompletion.cpp

@@ -3228,10 +3228,9 @@ class CompletionLookup final : public swift::VisibleDeclConsumer {
 
     auto floatType = context.getFloatDecl()->getDeclaredType();
     addFromProto(LK::ColorLiteral, "", [&](Builder &builder) {
-      builder.addLeftBracket();
-      builder.addTextChunk("#Color");
+      builder.addTextChunk("#colorLiteral");
       builder.addLeftParen();
-      builder.addCallParameter(context.getIdentifier("colorLiteralRed"),
+      builder.addCallParameter(context.getIdentifier("red"),
                                floatType, false, true);
       builder.addComma();
       builder.addCallParameter(context.getIdentifier("green"), floatType,
@@ -3243,20 +3242,15 @@ class CompletionLookup final : public swift::VisibleDeclConsumer {
       builder.addCallParameter(context.getIdentifier("alpha"), floatType,
                                false, true);
       builder.addRightParen();
-      builder.addTextChunk("#");
-      builder.addRightBracket();
     });
 
     auto stringType = context.getStringDecl()->getDeclaredType();
     addFromProto(LK::ImageLiteral, "", [&](Builder &builder) {
-      builder.addLeftBracket();
-      builder.addTextChunk("#Image");
+      builder.addTextChunk("#imageLiteral");
       builder.addLeftParen();
-      builder.addCallParameter(context.getIdentifier("imageLiteral"),
+      builder.addCallParameter(context.getIdentifier("resourceName"),
                                stringType, false, true);
       builder.addRightParen();
-      builder.addTextChunk("#");
-      builder.addRightBracket();
     });
 
     // Add tuple completion (item, item).

lib/IDE/SyntaxModel.cpp

@@ -84,10 +84,6 @@ SyntaxModelContext::SyntaxModelContext(SourceFile &SrcFile)
       Length = Tok.getLength();
 
       if (LiteralStartLoc.hasValue() && Length.hasValue()) {
-        // We are still inside an object literal until we hit a r_square_lit.
-        if (Tok.getKind() != tok::r_square_lit) {
-          continue;
-        }
         Kind = SyntaxNodeKind::ObjectLiteral;
         Nodes.emplace_back(Kind, CharSourceRange(SM, LiteralStartLoc.getValue(),
           Tok.getRange().getEnd()));
@@ -99,20 +95,23 @@ SyntaxModelContext::SyntaxModelContext(SourceFile &SrcFile)
 #define KEYWORD(X) case tok::kw_##X: Kind = SyntaxNodeKind::Keyword; break;
 #include "swift/Parse/Tokens.def"
 #undef KEYWORD
-      case tok::pound_selector:
-      case tok::pound_file:
-      case tok::pound_column:
-      case tok::pound_function:
-      case tok::pound_dsohandle:
+
+#define POUND_NORMAL_KEYWORD(Name) case tok::pound_##Name:
+#define POUND_OBJECT_LITERAL(Name, Desc, Proto) case tok::pound_##Name:
+#define POUND_OLD_OBJECT_LITERAL(Name, NewName, OldArg, NewArg) case tok::pound_##Name:
+#include "swift/Parse/Tokens.def"    
         Kind = SyntaxNodeKind::Keyword;
         break;
+
+#define POUND_CONFIG(Name) case tok::pound_##Name:
+#include "swift/Parse/Tokens.def"    
+        Kind = SyntaxNodeKind::BuildConfigKeyword;
+        break;
+
       case tok::pound_line:
         Kind = Tok.isAtStartOfLine() ? SyntaxNodeKind::BuildConfigKeyword :
                                        SyntaxNodeKind::Keyword;
         break;
-      case tok::pound_available:
-        Kind = SyntaxNodeKind::BuildConfigKeyword;
-        break;
       case tok::identifier:
         if (Tok.getText().startswith("<#"))
           Kind = SyntaxNodeKind::EditorPlaceholder;
@@ -189,11 +188,6 @@ SyntaxModelContext::SyntaxModelContext(SourceFile &SrcFile)
         break;
       }
 
-      case tok::l_square_lit: {
-        LiteralStartLoc = Loc;
-        continue;
-      }
-
       default:
         continue;
       }
@@ -490,11 +484,13 @@ std::pair<bool, Expr *> ModelASTWalker::walkToExprPre(Expr *E) {
   } else if (auto *ObjectE = dyn_cast<ObjectLiteralExpr>(E)) {
     SyntaxStructureNode SN;
     SN.Kind = SyntaxStructureKind::ObjectLiteralExpression;
-    SN.Range = charSourceRangeFromSourceRange(SM, E->getSourceRange());
-    SourceLoc NRStart = ObjectE->getNameLoc();
-    SourceLoc NREnd = NRStart.getAdvancedLoc(ObjectE->getName().getLength());
+    SN.Range = charSourceRangeFromSourceRange(SM, ObjectE->getSourceRange());
+    SourceLoc NRStart = ObjectE->getSourceLoc().getAdvancedLoc(1);    
+    SourceLoc NREnd =
+      NRStart.getAdvancedLoc(ObjectE->getLiteralKindRawName().size());
     SN.NameRange = CharSourceRange(SM, NRStart, NREnd);
-    SN.BodyRange = innerCharSourceRangeFromSourceRange(SM, E->getSourceRange());
+    SN.BodyRange =
+      innerCharSourceRangeFromSourceRange(SM, ObjectE->getSourceRange());
     pushStructureNode(SN, E);
 
   } else if (auto *ArrayE = dyn_cast<ArrayExpr>(E)) {

lib/Parse/Lexer.cpp

@@ -578,9 +578,11 @@ void Lexer::lexIdentifier() {
 /// lexHash - Handle #], #! for shebangs, and the family of #identifiers.
 void Lexer::lexHash() {
   const char *TokStart = CurPtr-1;
+
+  // NOTE: legacy punctuator.  Remove in the future.
   if (*CurPtr == ']') { // #]
-    CurPtr++;
-    return formToken(tok::r_square_lit, TokStart);
+     CurPtr++;
+     return formToken(tok::r_square_lit, TokStart);
   }
 
   // Allow a hashbang #! line at the beginning of the file.
@@ -1620,11 +1622,16 @@ void Lexer::lexImpl() {
   case '@': return formToken(tok::at_sign, TokStart);
   case '{': return formToken(tok::l_brace, TokStart);
   case '[': {
-    if (*CurPtr == '#') { // [#
-      CurPtr++;
-      return formToken(tok::l_square_lit, TokStart);
-    }
-    return formToken(tok::l_square, TokStart);
+     // NOTE: Legacy punctuator for old object literal syntax.
+     // Remove in the future.
+     if (*CurPtr == '#') { // [#
+       // NOTE: Do NOT include the '#' in the token, unlike in earlier
+       // versions of Swift that supported the old object literal syntax
+       // directly.  The '#' will be lexed as part of the object literal
+       // keyword token itself.
+       return formToken(tok::l_square_lit, TokStart);
+     }
+     return formToken(tok::l_square, TokStart);
   }
   case '(': return formToken(tok::l_paren, TokStart);
   case '}': return formToken(tok::r_brace, TokStart);