Add support for raw identifiers.

Raw identifiers are backtick-delimited identifiers that can contain any
non-identifier character other than the backtick itself, CR, LF, or other
non-printable ASCII code units, and which are also not composed entirely
of operator characters.

Author: allevato

include/swift/AST/ASTDemangler.h

@@ -306,6 +306,10 @@ class ASTBuilder {
 
   static GenericTypeDecl *getAcceptableTypeDeclCandidate(ValueDecl *decl,
                                               Demangle::Node::Kind kind);
+
+  /// Returns an identifier with the given name, automatically removing any
+  /// surrounding backticks that are present for raw identifiers.
+  Identifier getIdentifier(StringRef name);
 };
 
 SWIFT_END_INLINE_NAMESPACE

include/swift/AST/ASTPrinter.h

@@ -444,9 +444,9 @@ void printWithCompatibilityFeatureChecks(ASTPrinter &printer,
                                          Decl *decl,
                                          llvm::function_ref<void()> printBody);
 
-/// Determine whether we need to escape the given keyword within the
-/// given context, by wrapping it in backticks.
-bool escapeKeywordInContext(StringRef keyword, PrintNameContext context);
+/// Determine whether we need to escape the given name within the given
+/// context, by wrapping it in backticks.
+bool escapeIdentifierInContext(Identifier name, PrintNameContext context);
 
 } // namespace swift
 

include/swift/AST/Identifier.h

@@ -101,19 +101,22 @@ class Identifier {
 
   /// isOperator - Return true if this identifier is an operator, false if it is
   /// a normal identifier.
-  /// FIXME: We should maybe cache this.
   bool isOperator() const {
     if (empty())
       return false;
     if (isEditorPlaceholder())
       return false;
-    if ((unsigned char)Pointer[0] < 0x80)
-      return isOperatorStartCodePoint((unsigned char)Pointer[0]);
 
     // Handle the high unicode case out of line.
     return isOperatorSlow();
   }
 
+  /// Returns true if this identifier contains non-identifier characters and
+  /// must always be escaped with backticks, even in contexts were other
+  /// escaped identifiers could omit backticks (like keywords as argument
+  /// labels).
+  bool mustAlwaysBeEscaped() const;
+
   bool isArithmeticOperator() const {
     return is("+") || is("-") || is("*") || is("/") || is("%");
   }
@@ -350,6 +353,10 @@ class DeclBaseName {
     return !isSpecial() && getIdentifier().isOperator();
   }
 
+  bool mustAlwaysBeEscaped() const {
+    return !isSpecial() && getIdentifier().mustAlwaysBeEscaped();
+  }
+
   bool isEditorPlaceholder() const {
     return !isSpecial() && getIdentifier().isEditorPlaceholder();
   }
@@ -571,7 +578,12 @@ class DeclName {
   bool isOperator() const {
     return getBaseName().isOperator();
   }
-  
+
+  /// True if this name is an escaped identifier.
+  bool mustAlwaysBeEscaped() const {
+    return getBaseName().mustAlwaysBeEscaped();
+  }
+
   /// True if this name should be found by a decl ref or member ref under the
   /// name specified by 'refName'.
   ///
@@ -728,6 +740,8 @@ class DeclNameRef {
     return FullName.isOperator();
   }
 
+  bool mustAlwaysBeEscaped() const { return FullName.mustAlwaysBeEscaped(); }
+
   bool isCompoundName() const {
     return FullName.isCompoundName();
   }

include/swift/Basic/Mangler.h

@@ -114,6 +114,13 @@ class Mangler {
     print(llvm::dbgs());
   }
 
+  /// Appends the given raw identifier to the buffer in the form required to
+  /// mangle it. This handles the transformations needed for such identifiers
+  /// to retain compatibility with older runtimes.
+  static void
+  appendRawIdentifierForRuntime(StringRef ident,
+                                llvm::SmallVectorImpl<char> &buffer);
+
 protected:
   /// Removes the last characters of the buffer by setting it's size to a
   /// smaller value.
@@ -143,7 +150,7 @@ class Mangler {
   SWIFT_DEBUG_DUMP;
 
   /// Appends a mangled identifier string.
-  void appendIdentifier(StringRef ident);
+  void appendIdentifier(StringRef ident, bool allowRawIdentifiers = true);
 
   // NOTE: the addSubstitution functions perform the value computation before
   // the assignment because there is no sequence point synchronising the

include/swift/Demangling/ManglingUtils.h

@@ -58,10 +58,16 @@ inline bool isWordEnd(char ch, char prevCh) {
   return false;
 }
 
+/// Returns true if \p ch is a valid character which may appear at the start
+/// of a symbol mangling.
+inline bool isValidSymbolStart(char ch) {
+  return isLetter(ch) || ch == '_' || ch == '$';
+}
+
 /// Returns true if \p ch is a valid character which may appear in a symbol
-/// mangling.
+/// mangling anywhere other than the first character.
 inline bool isValidSymbolChar(char ch) {
-  return isLetter(ch) || isDigit(ch) || ch == '_' || ch == '$';
+  return isValidSymbolStart(ch) || isDigit(ch);
 }
 
 /// Returns true if \p str contains any character which may not appear in a

include/swift/IDE/CompletionLookup.h

@@ -335,6 +335,8 @@ class CompletionLookup final : public swift::VisibleDeclConsumer {
   void addValueBaseName(CodeCompletionResultBuilder &Builder,
                         DeclBaseName Name);
 
+  void addIdentifier(CodeCompletionResultBuilder &Builder, Identifier Name);
+
   void addLeadingDot(CodeCompletionResultBuilder &Builder);
 
   void addTypeAnnotation(CodeCompletionResultBuilder &Builder, Type T,

include/swift/Parse/Lexer.h

@@ -395,6 +395,11 @@ class Lexer {
   /// identifier, without escaping characters.
   static bool isIdentifier(StringRef identifier);
 
+  // Returns true if the given string is a raw identifier that must always
+  // be escaped by backticks when printing it back in source form or writing
+  // its name into runtime metadata.
+  static bool identifierMustAlwaysBeEscaped(StringRef str);
+
   /// Determine the token kind of the string, given that it is a valid
   /// non-operator identifier. Return tok::identifier if the string is not a
   /// reserved word.

lib/AST/ASTDemangler.cpp

@@ -94,7 +94,7 @@ TypeDecl *ASTBuilder::createTypeDecl(NodePointer node) {
     if (proto == nullptr)
       return nullptr;
 
-    auto name = Ctx.getIdentifier(node->getChild(1)->getText());
+    auto name = getIdentifier(node->getChild(1)->getText());
     return proto->getAssociatedType(name);
   }
 
@@ -110,10 +110,9 @@ ASTBuilder::createBuiltinType(StringRef builtinName,
 
     StringRef strippedName =
         builtinName.drop_front(BUILTIN_TYPE_NAME_PREFIX.size());
-    Ctx.TheBuiltinModule->lookupValue(Ctx.getIdentifier(strippedName),
-                                      NLKind::QualifiedLookup,
-                                      decls);
-    
+    Ctx.TheBuiltinModule->lookupValue(getIdentifier(strippedName),
+                                      NLKind::QualifiedLookup, decls);
+
     if (decls.size() == 1 && isa<TypeDecl>(decls[0]))
       return cast<TypeDecl>(decls[0])->getDeclaredInterfaceType();
   }
@@ -348,7 +347,7 @@ Type ASTBuilder::createTupleType(ArrayRef<Type> eltTypes, ArrayRef<StringRef> la
   for (unsigned i : indices(eltTypes)) {
     Identifier label;
     if (!labels[i].empty())
-      label = Ctx.getIdentifier(labels[i]);
+      label = getIdentifier(labels[i]);
     elements.emplace_back(eltTypes[i], label);
   }
 
@@ -408,7 +407,7 @@ Type ASTBuilder::createFunctionType(
     if (!type->isMaterializable())
       return Type();
 
-    auto label = Ctx.getIdentifier(param.getLabel());
+    auto label = getIdentifier(param.getLabel());
     auto flags = param.getFlags();
     auto ownership =
       ParamDecl::getParameterSpecifierForValueOwnership(asValueOwnership(flags.getOwnership()));
@@ -878,7 +877,7 @@ Type ASTBuilder::createGenericTypeParameterType(unsigned depth,
 
 Type ASTBuilder::createDependentMemberType(StringRef member,
                                            Type base) {
-  auto identifier = Ctx.getIdentifier(member);
+  auto identifier = getIdentifier(member);
 
   if (auto *archetype = base->getAs<ArchetypeType>()) {
       if (Type memberType = archetype->getNestedTypeByName(identifier))
@@ -895,7 +894,7 @@ Type ASTBuilder::createDependentMemberType(StringRef member,
 Type ASTBuilder::createDependentMemberType(StringRef member,
                                            Type base,
                                            ProtocolDecl *protocol) {
-  auto identifier = Ctx.getIdentifier(member);
+  auto identifier = getIdentifier(member);
 
   if (auto *archetype = base->getAs<ArchetypeType>()) {
     if (auto assocType = protocol->getAssociatedType(identifier))
@@ -1135,7 +1134,7 @@ ASTBuilder::getAcceptableTypeDeclCandidate(ValueDecl *decl,
 
 DeclContext *ASTBuilder::getNotionalDC() {
   if (!NotionalDC) {
-    NotionalDC = ModuleDecl::createEmpty(Ctx.getIdentifier(".RemoteAST"), Ctx);
+    NotionalDC = ModuleDecl::createEmpty(getIdentifier(".RemoteAST"), Ctx);
     NotionalDC = new (Ctx) TopLevelCodeDecl(NotionalDC);
   }
   return NotionalDC;
@@ -1308,7 +1307,7 @@ ASTBuilder::findDeclContext(NodePointer node) {
                  Demangle::Node::Kind::PrivateDeclName) {
       name = declNameNode->getChild(1)->getText();
       privateDiscriminator =
-        Ctx.getIdentifier(declNameNode->getChild(0)->getText());
+          getIdentifier(declNameNode->getChild(0)->getText());
 
     } else if (declNameNode->getKind() ==
                  Demangle::Node::Kind::RelatedEntityDeclName) {
@@ -1336,14 +1335,14 @@ ASTBuilder::findDeclContext(NodePointer node) {
         return nullptr;
 
       for (auto *module : potentialModules)
-        if (auto typeDecl = findTypeDecl(module, Ctx.getIdentifier(name),
+        if (auto typeDecl = findTypeDecl(module, getIdentifier(name),
                                          privateDiscriminator, node->getKind()))
           return typeDecl;
       return nullptr;
     }
 
     if (auto *dc = findDeclContext(child))
-      if (auto typeDecl = findTypeDecl(dc, Ctx.getIdentifier(name),
+      if (auto typeDecl = findTypeDecl(dc, getIdentifier(name),
                                        privateDiscriminator, node->getKind()))
         return typeDecl;
 
@@ -1542,7 +1541,7 @@ GenericTypeDecl *ASTBuilder::findForeignTypeDecl(StringRef name,
                                     found);
       break;
     }
-    importer->lookupValue(Ctx.getIdentifier(name), consumer);
+    importer->lookupValue(getIdentifier(name), consumer);
     if (consumer.Result)
       consumer.Result = getAcceptableTypeDeclCandidate(consumer.Result, kind);
     break;
@@ -1552,3 +1551,28 @@ GenericTypeDecl *ASTBuilder::findForeignTypeDecl(StringRef name,
 
   return consumer.Result;
 }
+
+Identifier ASTBuilder::getIdentifier(StringRef name) {
+  if (name.size() > 1 && name.front() == '`' && name.back() == '`') {
+    // Raw identifiers have backticks affixed before mangling. We need to
+    // remove those before creating the Identifier for the AST, which doesn't
+    // encode the backticks.
+    std::string fixedName;
+    for (size_t i = 1; i < name.size() - 1; ++i) {
+      unsigned char ch = name[i];
+      // Raw identifiers have the space (U+0020) replaced with a non-breaking
+      // space (U+00A0, UTF-8: 0xC2 0xA0) in their mangling so that parts of
+      // the runtime that use space as a delimiter remain compatible with
+      // these identifiers. Flip it back.
+      if (ch == 0xc2 && i < name.size() - 2 &&
+          (unsigned char)name[i + 1] == 0xa0) {
+        fixedName.push_back(' ');
+        ++i;
+      } else {
+        fixedName.push_back(ch);
+      }
+    }
+    return Ctx.getIdentifier(fixedName);
+  }
+  return Ctx.getIdentifier(name);
+}

lib/AST/ASTMangler.cpp

@@ -1146,7 +1146,7 @@ void ASTMangler::appendDeclName(const ValueDecl *decl, DeclBaseName name) {
            "synthesized type's original name must be a valid Swift identifier");
     appendIdentifier(synthesizedTypeAttr->originalTypeName);
   } else if (name.isOperator()) {
-    appendIdentifier(translateOperator(name.getIdentifier().str()));
+    appendIdentifier(translateOperator(name.getIdentifier().str()), /*allowRawIdentifiers=*/ false);
     switch (decl->getAttrs().getUnaryOperatorKind()) {
       case UnaryOperatorKind::Prefix:
         appendOperator("op");
@@ -4719,7 +4719,7 @@ void ASTMangler::appendMacroExpansionContext(
     appendIdentifier(origDC->getParentModule()->getName().str());
 
     auto *SF = origDC->getParentSourceFile();
-    appendIdentifier(llvm::sys::path::filename(SF->getFilename()));
+    appendIdentifier(llvm::sys::path::filename(SF->getFilename()), /*allowRawIdentifiers=*/false);
 
     auto lineColumn = sourceMgr.getLineAndColumnInBuffer(loc);
     appendOperator("fMX", Index(lineColumn.first), Index(lineColumn.second));

lib/AST/ASTPrinter.cpp

@@ -558,10 +558,9 @@ ASTPrinter &operator<<(ASTPrinter &printer, tok keyword) {
 }
 
 /// Determine whether to escape the given keyword in the given context.
-bool swift::escapeKeywordInContext(
-    StringRef keyword,
-    PrintNameContext context
-) {
+bool swift::escapeIdentifierInContext(Identifier name,
+                                      PrintNameContext context) {
+  StringRef keyword = name.str();
   bool isKeyword = llvm::StringSwitch<bool>(keyword)
 #define KEYWORD(KW) \
       .Case(#KW, true)
@@ -571,7 +570,7 @@ bool swift::escapeKeywordInContext(
   switch (context) {
   case PrintNameContext::Normal:
   case PrintNameContext::Attribute:
-    return isKeyword;
+    return isKeyword || name.mustAlwaysBeEscaped();
   case PrintNameContext::Keyword:
   case PrintNameContext::IntroducerKeyword:
     return false;
@@ -581,12 +580,12 @@ bool swift::escapeKeywordInContext(
     return isKeyword && keyword != "Self";
 
   case PrintNameContext::TypeMember:
-    return isKeyword || !canBeMemberName(keyword);
+    return isKeyword || !canBeMemberName(keyword) || name.mustAlwaysBeEscaped();
 
   case PrintNameContext::FunctionParameterExternal:
   case PrintNameContext::FunctionParameterLocal:
   case PrintNameContext::TupleElement:
-    return !canBeArgumentLabel(keyword);
+    return !canBeArgumentLabel(keyword) || name.mustAlwaysBeEscaped();
   }
 
   llvm_unreachable("Unhandled PrintNameContext in switch.");
@@ -601,12 +600,12 @@ void ASTPrinter::printName(Identifier Name, PrintNameContext Context) {
     return;
   }
 
-  bool shouldEscapeKeyword = escapeKeywordInContext(Name.str(), Context);
+  bool shouldEscapeIdentifier = escapeIdentifierInContext(Name, Context);
 
-  if (shouldEscapeKeyword)
+  if (shouldEscapeIdentifier)
     *this << "`";
   *this << Name.str();
-  if (shouldEscapeKeyword)
+  if (shouldEscapeIdentifier)
     *this << "`";
 
   printNamePost(Context);

lib/AST/Attr.cpp

@@ -1533,7 +1533,8 @@ bool DeclAttribute::printImpl(ASTPrinter &Printer, const PrintOptions &Options,
               StringRef nameText = name.getName().getString(buffer);
               bool shouldEscape =
                   !name.getName().isSpecial() &&
-                  (escapeKeywordInContext(nameText, PrintNameContext::Normal) ||
+                  (escapeIdentifierInContext(name.getName().getBaseIdentifier(),
+                                             PrintNameContext::Normal) ||
                    nameText == "$");
               Printer << "(";
               if (shouldEscape)

lib/AST/Identifier.cpp

@@ -78,16 +78,10 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, swift::ObjCSelector S) {
   return OS;
 }
 
-bool Identifier::isOperatorSlow() const {
-  StringRef data = str();
-  auto *s = reinterpret_cast<llvm::UTF8 const *>(data.begin()),
-  *end = reinterpret_cast<llvm::UTF8 const *>(data.end());
-  llvm::UTF32 codePoint;
-  llvm::ConversionResult res =
-    llvm::convertUTF8Sequence(&s, end, &codePoint, llvm::strictConversion);
-  assert(res == llvm::conversionOK && "invalid UTF-8 in identifier?!");
-  (void)res;
-  return !empty() && isOperatorStartCodePoint(codePoint);
+bool Identifier::isOperatorSlow() const { return Lexer::isOperator(str()); }
+
+bool Identifier::mustAlwaysBeEscaped() const {
+  return Lexer::identifierMustAlwaysBeEscaped(str());
 }
 
 int Identifier::compare(Identifier other) const {