[Macros] [SE-0382] Implement macros defined as expansions of other macros

include/swift/AST/ASTScope.h

@@ -1215,6 +1215,26 @@ class MacroDeclScope final : public ASTScopeImpl {
 
 protected:
   NullablePtr<const GenericParamList> genericParams() const override;
+  bool lookupLocalsOrMembers(DeclConsumer) const override;
+};
+
+/// The scope introduced for the definition of a macro, which follows the `=`.
+class MacroDefinitionScope final : public ASTScopeImpl {
+public:
+  Expr *const definition;
+
+  MacroDefinitionScope(Expr *definition) : definition(definition) {}
+
+  virtual ~MacroDefinitionScope() {}
+  SourceRange
+  getSourceRangeOfThisASTNode(bool omitAssertions = false) const override;
+  std::string getClassName() const override;
+
+private:
+  void expandAScopeThatDoesNotCreateANewInsertionPoint(ScopeCreator &);
+
+protected:
+  ASTScopeImpl *expandSpecifically(ScopeCreator &scopeCreator) override;
 };
 
 class AbstractStmtScope : public ASTScopeImpl {

include/swift/AST/CASTBridging.h

@@ -102,6 +102,16 @@ typedef enum ENUM_EXTENSIBILITY_ATTR(open) BridgedDiagnosticSeverity : long {
 
 typedef void* BridgedDiagnostic;
 
+typedef enum ENUM_EXTENSIBILITY_ATTR(open) BridgedMacroDefinitionKind : ptrdiff_t {
+  /// An expanded macro.
+  BridgedExpandedMacro = 0,
+  /// An external macro, spelled with either the old spelling (Module.Type)
+  /// or the new spelling `#externalMacro(module: "Module", type: "Type")`.
+  BridgedExternalMacro,
+  /// The builtin definition for "externalMacro".
+  BridgedBuiltinExternalMacro
+} BridgedMacroDefinitionKind;
+
 #ifdef __cplusplus
 extern "C" {
 

include/swift/AST/DiagnosticsSema.def

@@ -7005,13 +7005,6 @@ ERROR(macro_unsupported,none,
       "macros are not supported in this compiler", ())
 ERROR(macro_recursive,none,
       "recursive expansion of macro %0", (DeclName))
-WARNING(macro_definition_old_style,none,
-        "external macro definitions are now written using #externalMacro", ())
-WARNING(macro_definition_unknown_builtin,none,
-        "ignoring definition of unknown builtin macro %0", (Identifier))
-ERROR(macro_definition_not_expansion,none,
-      "macro must itself be defined by a macro expansion such as "
-      "'#externalMacro(...)'", ())
 ERROR(macro_definition_unsupported,none,
       "macro definitions other than '#externalMacro(...)' are unsupported", ())
 ERROR(external_macro_arg_not_type_name,none,

include/swift/AST/MacroDefinition.h

@@ -22,6 +22,8 @@
 
 namespace swift {
 
+class ASTContext;
+
 /// A reference to an external macro definition that is understood by ASTGen.
 struct ExternalMacroDefinition {
   enum class PluginKind {
@@ -46,6 +48,35 @@ enum class BuiltinMacroKind: uint8_t {
   ExternalMacro,
 };
 
+/// A single replacement
+struct ExpandedMacroReplacement {
+  unsigned startOffset, endOffset;
+  unsigned parameterIndex;
+};
+
+/// An expansion of another macro.
+class ExpandedMacroDefinition {
+  friend class MacroDefinition;
+
+  /// The expansion text, ASTContext-allocated.
+  StringRef expansionText;
+
+  /// The macro replacements, ASTContext-allocated.
+  ArrayRef<ExpandedMacroReplacement> replacements;
+
+  ExpandedMacroDefinition(
+    StringRef expansionText,
+    ArrayRef<ExpandedMacroReplacement> replacements
+  ) : expansionText(expansionText), replacements(replacements) { }
+
+public:
+  StringRef getExpansionText() const { return expansionText; }
+
+  ArrayRef<ExpandedMacroReplacement> getReplacements() const {
+    return replacements;
+  }
+};
+
 /// Provides the definition of a macro.
 class MacroDefinition {
 public:
@@ -63,6 +94,9 @@ class MacroDefinition {
 
     /// A builtin macro definition, which has a separate builtin kind.
     Builtin,
+
+    /// A macro that is defined as an expansion of another macro.
+    Expanded,
   };
 
   Kind kind;
@@ -71,6 +105,7 @@ class MacroDefinition {
   union Data {
     ExternalMacroReference external;
     BuiltinMacroKind builtin;
+    ExpandedMacroDefinition expanded;
 
     Data() : builtin(BuiltinMacroKind::ExternalMacro) { }
   } data;
@@ -85,6 +120,10 @@ class MacroDefinition {
     data.builtin = builtinKind;
   }
 
+  MacroDefinition(ExpandedMacroDefinition expanded) : kind(Kind::Expanded) {
+    data.expanded = expanded;
+  }
+
 public:
   static MacroDefinition forInvalid() {
     return MacroDefinition(Kind::Invalid);
@@ -105,6 +144,13 @@ class MacroDefinition {
     return MacroDefinition(builtinKind);
   }
 
+  /// Create a representation of an expanded macro definition.
+  static MacroDefinition forExpanded(
+      ASTContext &ctx,
+      StringRef expansionText,
+      ArrayRef<ExpandedMacroReplacement> replacements
+  );
+
   /// Retrieve the external macro being referenced.
   ExternalMacroReference getExternalMacro() const {
     assert(kind == Kind::External);
@@ -117,6 +163,11 @@ class MacroDefinition {
     return data.builtin;
   }
 
+  ExpandedMacroDefinition getExpanded() const {
+    assert(kind == Kind::Expanded);
+    return data.expanded;
+  }
+
   operator Kind() const { return kind; }
 };
 

include/swift/Basic/Features.def

@@ -94,11 +94,11 @@ SUPPRESSIBLE_LANGUAGE_FEATURE(PrimaryAssociatedTypes2, 346, "Primary associated
 SUPPRESSIBLE_LANGUAGE_FEATURE(UnavailableFromAsync, 0, "@_unavailableFromAsync", true)
 SUPPRESSIBLE_LANGUAGE_FEATURE(NoAsyncAvailability, 340, "@available(*, noasync)", true)
 LANGUAGE_FEATURE(BuiltinIntLiteralAccessors, 368, "Builtin.IntLiteral accessors", true)
-LANGUAGE_FEATURE(Macros, 0, "Macros", true)
+LANGUAGE_FEATURE(Macros, 0, "Macros", hasSwiftSwiftParser)
 SUPPRESSIBLE_LANGUAGE_FEATURE(
     FreestandingExpressionMacros, 382, "Expression macros",
-    true)
-LANGUAGE_FEATURE(AttachedMacros, 389, "Attached macros", true)
+        hasSwiftSwiftParser)
+LANGUAGE_FEATURE(AttachedMacros, 389, "Attached macros", hasSwiftSwiftParser)
 LANGUAGE_FEATURE(MoveOnly, 390, "noncopyable types", true)
 
 UPCOMING_FEATURE(ConciseMagicFile, 274, 6)

include/swift/Basic/StringExtras.h

@@ -490,7 +490,7 @@ class NullTerminatedStringRef {
 
   /// Create a null-terminated string, copying \p Str into \p A .
   template <typename Allocator>
-  NullTerminatedStringRef(StringRef Str, Allocator &A) : Ref("") {
+  NullTerminatedStringRef(StringRef Str, Allocator &A) : Ref() {
     if (Str.empty())
       return;
 

lib/AST/ASTMangler.cpp

@@ -3869,12 +3869,14 @@ void ASTMangler::appendMacroExpansionContext(
       baseName = expr->getMacroName().getBaseName();
       discriminator = expr->getDiscriminator();
       role = MacroRole::Expression;
+      outerExpansionDC = expr->getDeclContext();
     } else {
       auto decl = cast<MacroExpansionDecl>(parent.get<Decl *>());
       outerExpansionLoc = decl->getLoc();
       baseName = decl->getMacroName().getBaseName();
       discriminator = decl->getDiscriminator();
       role = MacroRole::Declaration;
+      outerExpansionDC = decl->getDeclContext();
     }
     break;
   }

lib/AST/ASTPrinter.cpp

@@ -4758,8 +4758,8 @@ void PrintAST::visitMacroDecl(MacroDecl *decl) {
 
       case MacroDefinition::Kind::External: {
         auto external = def.getExternalMacro();
-        Printer << " = #externalMacro(module: \"" << external.moduleName << "\", "
-                << "type: \"" << external.macroTypeName << "\")";
+        Printer << " = #externalMacro(module: \"" << external.moduleName
+                << "\", " << "type: \"" << external.macroTypeName << "\")";
         break;
       }
 
@@ -4771,6 +4771,10 @@ void PrintAST::visitMacroDecl(MacroDecl *decl) {
           break;
         }
         break;
+
+      case MacroDefinition::Kind::Expanded:
+        Printer << " = " << def.getExpanded().getExpansionText();
+        break;
       }
     }
   }

lib/AST/ASTScope.cpp

@@ -151,6 +151,7 @@ DEFINE_GET_CLASS_NAME(DifferentiableAttributeScope)
 DEFINE_GET_CLASS_NAME(SubscriptDeclScope)
 DEFINE_GET_CLASS_NAME(EnumElementScope)
 DEFINE_GET_CLASS_NAME(MacroDeclScope)
+DEFINE_GET_CLASS_NAME(MacroDefinitionScope)
 DEFINE_GET_CLASS_NAME(IfStmtScope)
 DEFINE_GET_CLASS_NAME(WhileStmtScope)
 DEFINE_GET_CLASS_NAME(GuardStmtScope)

lib/AST/ASTScopeCreation.cpp

@@ -723,6 +723,7 @@ NO_NEW_INSERTION_POINT(IfStmtScope)
 NO_NEW_INSERTION_POINT(RepeatWhileScope)
 NO_NEW_INSERTION_POINT(SubscriptDeclScope)
 NO_NEW_INSERTION_POINT(MacroDeclScope)
+NO_NEW_INSERTION_POINT(MacroDefinitionScope)
 NO_NEW_INSERTION_POINT(SwitchStmtScope)
 NO_NEW_INSERTION_POINT(WhileStmtScope)
 
@@ -1115,9 +1116,19 @@ void MacroDeclScope::expandAScopeThatDoesNotCreateANewInsertionPoint(
   auto *leaf = scopeCreator.addNestedGenericParamScopesToTree(
       decl, getPotentiallyOpaqueGenericParams(decl), this);
   if (decl->parameterList) {
-    scopeCreator.constructExpandAndInsert<ParameterListScope>(
+    leaf = scopeCreator.constructExpandAndInsert<ParameterListScope>(
         leaf, decl->parameterList, nullptr);
   }
+  if (auto def = decl->definition) {
+    scopeCreator
+      .constructExpandAndInsert<MacroDefinitionScope>(leaf, def);
+  }
+}
+
+void
+MacroDefinitionScope::expandAScopeThatDoesNotCreateANewInsertionPoint(
+    ScopeCreator &scopeCreator) {
+  scopeCreator.addToScopeTree(ASTNode(definition), this);
 }
 
 void CaptureListScope::expandAScopeThatDoesNotCreateANewInsertionPoint(

lib/AST/ASTScopeLookup.cpp

@@ -260,6 +260,17 @@ NullablePtr<const GenericParamList> MacroDeclScope::genericParams() const {
   return decl->getParsedGenericParams();
 }
 
+bool MacroDeclScope::lookupLocalsOrMembers(
+    DeclConsumer consumer) const {
+  if (auto *paramList = decl->parameterList) {
+    for (auto *paramDecl : *paramList)
+      if (consumer.consume({paramDecl}))
+        return true;
+  }
+
+  return false;
+}
+
 #pragma mark lookInMyGenericParameters
 
 std::pair<bool, NullablePtr<const GenericParamList>>

lib/AST/ASTScopeSourceRange.cpp

@@ -143,6 +143,11 @@ SourceRange MacroDeclScope::getSourceRangeOfThisASTNode(
   return decl->getSourceRangeIncludingAttrs();
 }
 
+SourceRange MacroDefinitionScope::getSourceRangeOfThisASTNode(
+    const bool omitAssertions) const {
+  return definition->getSourceRange();
+}
+
 SourceRange
 EnumElementScope::getSourceRangeOfThisASTNode(const bool omitAssertions) const {
   return decl->getSourceRange();

lib/AST/ASTVerifier.cpp

@@ -477,7 +477,7 @@ class Verifier : public ASTWalker {
         // For @objc enums, we serialize the pre-type-checked integer
         // literal raw values, and thus when they are deserialized
         // they do not have a type on them.
-        if (!isa<IntegerLiteralExpr>(E)) {
+        if (!isa<IntegerLiteralExpr>(E) && !isa<MacroExpansionExpr>(E)) {
           Out << "expression has no type\n";
           E->dump(Out);
           abort();
@@ -769,6 +769,7 @@ class Verifier : public ASTWalker {
     FUNCTION_LIKE(FuncDecl)
     FUNCTION_LIKE(EnumElementDecl)
     FUNCTION_LIKE(SubscriptDecl)
+    FUNCTION_LIKE(MacroDecl)
     TYPE_LIKE(NominalTypeDecl)
     TYPE_LIKE(ExtensionDecl)
 

lib/AST/ASTWalker.cpp

@@ -422,6 +422,18 @@ class Traversal : public ASTVisitor<Traversal, Expr*, Stmt*,
         return true;
     }
 
+    if (auto def = MD->definition) {
+      // Don't walk into unchecked definitions.
+      if (auto expansion = dyn_cast<MacroExpansionExpr>(def)) {
+        if (!expansion->getType().isNull()) {
+          if (auto newDef = doIt(def))
+            MD->definition = newDef;
+          else
+            return true;
+        }
+      }
+    }
+
     // Visit trailing requirements
     if (WalkGenerics && visitTrailingRequirements(MD))
        return true;

lib/AST/Decl.cpp

@@ -10326,6 +10326,15 @@ Optional<BuiltinMacroKind> MacroDecl::getBuiltinKind() const {
   return def.getBuiltinKind();
 }
 
+MacroDefinition MacroDefinition::forExpanded(
+    ASTContext &ctx,
+    StringRef expansionText,
+    ArrayRef<ExpandedMacroReplacement> replacements
+) {
+  return ExpandedMacroDefinition{ctx.AllocateCopy(expansionText),
+                                 ctx.AllocateCopy(replacements)};
+}
+
 MacroExpansionDecl::MacroExpansionDecl(
     DeclContext *dc, SourceLoc poundLoc, DeclNameRef macro,
     DeclNameLoc macroLoc, SourceLoc leftAngleLoc,

lib/AST/DistributedDecl.cpp

@@ -56,7 +56,6 @@
 #include "clang/AST/Attr.h"
 #include "clang/AST/DeclObjC.h"
 
-#include "InlinableText.h"
 #include <algorithm>
 
 using namespace swift;

lib/ASTGen/Sources/ASTGen/Diagnostics.swift

@@ -170,10 +170,13 @@ extension SourceManager {
 
       switch change {
       case .replace(let oldNode, let newNode):
-        replaceStartLoc = cxxSourceLocation(for: oldNode)
+        replaceStartLoc = cxxSourceLocation(
+          for: oldNode,
+          at: oldNode.positionAfterSkippingLeadingTrivia
+        )
         replaceEndLoc = cxxSourceLocation(
           for: oldNode,
-          at: oldNode.endPosition
+          at: oldNode.endPositionBeforeTrailingTrivia
         )
         newText = newNode.description