[Macros] Add (de-)serialization for expanded macro definitions

(cherry picked from commit 38230b7)

Author: DougGregor

lib/AST/ASTVerifier.cpp

@@ -769,6 +769,7 @@ class Verifier : public ASTWalker {
     FUNCTION_LIKE(FuncDecl)
     FUNCTION_LIKE(EnumElementDecl)
     FUNCTION_LIKE(SubscriptDecl)
+    FUNCTION_LIKE(MacroDecl)
     TYPE_LIKE(NominalTypeDecl)
     TYPE_LIKE(ExtensionDecl)
 

lib/Serialization/DeclTypeRecordNodes.def

@@ -184,6 +184,8 @@ OTHER(PRIVATE_DISCRIMINATOR, 128)
 OTHER(FILENAME_FOR_PRIVATE, 129)
 
 OTHER(DESERIALIZATION_SAFETY, 130)
+OTHER(EXPANDED_MACRO_DEFINITION, 131)
+OTHER(EXPANDED_MACRO_REPLACEMENTS, 132)
 
 // 140 is unused; was ABSTRACT_PROTOCOL_CONFORMANCE
 OTHER(NORMAL_PROTOCOL_CONFORMANCE, 141)

lib/Serialization/Deserialization.cpp

@@ -4676,6 +4676,7 @@ class DeclDeserializer {
     uint8_t rawAccessLevel;
     unsigned numArgNames;
     unsigned builtinID;
+    uint8_t hasExpandedMacroDefinition;
     IdentifierID externalModuleNameID;
     IdentifierID externalMacroTypeNameID;
 
@@ -4689,6 +4690,7 @@ class DeclDeserializer {
                                          rawAccessLevel,
                                          numArgNames,
                                          builtinID,
+                                         hasExpandedMacroDefinition,
                                          externalModuleNameID,
                                          externalMacroTypeNameID,
                                          argNameAndDependencyIDs);
@@ -4782,6 +4784,58 @@ class DeclDeserializer {
             MF.getIdentifier(externalMacroTypeNameID)
          )
       );
+    } else if (hasExpandedMacroDefinition) {
+      // Macro expansion definition block.
+      llvm::BitstreamEntry entry =
+          MF.fatalIfUnexpected(MF.DeclTypeCursor.advance(AF_DontPopBlockAtEnd));
+      if (entry.Kind != llvm::BitstreamEntry::Record)
+        return macro;
+
+      SmallVector<uint64_t, 16> scratch;
+      scratch.clear();
+      StringRef expansionText;
+      unsigned recordID = MF.fatalIfUnexpected(
+          MF.DeclTypeCursor.readRecord(entry.ID, scratch, &expansionText));
+      if (recordID != decls_block::EXPANDED_MACRO_DEFINITION)
+        return macro;
+
+      uint8_t hasReplacements;
+      decls_block::ExpandedMacroDefinitionLayout::readRecord(
+          scratch, hasReplacements);
+
+      // Macro replacements block.
+      SmallVector<ExpandedMacroReplacement, 2> replacements;
+      if (hasReplacements) {
+        llvm::BitstreamEntry entry =
+            MF.fatalIfUnexpected(
+              MF.DeclTypeCursor.advance(AF_DontPopBlockAtEnd));
+        if (entry.Kind == llvm::BitstreamEntry::Record) {
+          scratch.clear();
+          unsigned recordID = MF.fatalIfUnexpected(
+              MF.DeclTypeCursor.readRecord(entry.ID, scratch, &blobData));
+          if (recordID != decls_block::EXPANDED_MACRO_REPLACEMENTS)
+            return macro;
+
+          ArrayRef<uint64_t> serializedReplacements;
+          decls_block::ExpandedMacroReplacementsLayout::readRecord(
+              scratch, serializedReplacements);
+          if (serializedReplacements.size() % 3 == 0) {
+            for (unsigned i : range(0, serializedReplacements.size() / 3)) {
+              ExpandedMacroReplacement replacement{
+                static_cast<unsigned>(serializedReplacements[3*i]),
+                static_cast<unsigned>(serializedReplacements[3*i + 1]),
+                static_cast<unsigned>(serializedReplacements[3*i + 2])
+              };
+              replacements.push_back(replacement);
+            }
+          }
+        }
+      }
+
+      ctx.evaluator.cacheOutput(
+          MacroDefinitionRequest{macro},
+          MacroDefinition::forExpanded(ctx, expansionText, replacements)
+      );
     }
 
     return macro;

lib/Serialization/ModuleFormat.h

@@ -58,7 +58,7 @@ const uint16_t SWIFTMODULE_VERSION_MAJOR = 0;
 /// describe what change you made. The content of this comment isn't important;
 /// it just ensures a conflict if two people change the module format.
 /// Don't worry about adhering to the 80-column limit for this line.
-const uint16_t SWIFTMODULE_VERSION_MINOR = 756; // build complexEquality executor builtin
+const uint16_t SWIFTMODULE_VERSION_MINOR = 757; // expanded macro definitions
 
 /// A standard hash seed used for all string hashes in a serialized module.
 ///
@@ -1726,13 +1726,30 @@ namespace decls_block {
     AccessLevelField, // access level
     BCVBR<5>,    // number of parameter name components
     BCVBR<3>,    // builtin macro definition ID
+    BCFixed<1>,  // whether it has an expanded macro definition
     IdentifierIDField, // external module name, for external macros
     IdentifierIDField,  // external type name, for external macros
     BCArray<IdentifierIDField> // name components,
                                // followed by TypeID dependencies
     // The record is trailed by:
     // - its generic parameters, if any
     // - parameter list, if present
+    // - expanded macro definition, if needed.
+  >;
+
+  /// The expanded macro definition text.
+  using ExpandedMacroDefinitionLayout = BCRecordLayout<
+    EXPANDED_MACRO_DEFINITION,
+    BCFixed<1>, // whether it has replacements
+    BCBlob // expansion text
+    // potentially trailed by the expanded macro replacements
+  >;
+
+  /// The replacements to be performed for an expanded macro definition.
+  using ExpandedMacroReplacementsLayout = BCRecordLayout<
+    EXPANDED_MACRO_REPLACEMENTS,
+    BCArray<BCVBR<6>> // a set of replacement triples (start offset,
+                      // end offset, parameter index)
   >;
 
   using InlinableBodyTextLayout = BCRecordLayout<

lib/Serialization/Serialization.cpp

@@ -4590,8 +4590,10 @@ class Serializer::DeclSerializer : public DeclVisitor<DeclSerializer> {
     Type resultType = macro->getResultInterfaceType();
 
     uint8_t builtinID = 0;
+    uint8_t hasExpandedDefinition = 0;
     IdentifierID externalModuleNameID = 0;
     IdentifierID externalMacroTypeNameID = 0;
+    Optional<ExpandedMacroDefinition> expandedDef;
     auto def = macro->getDefinition();
     switch (def.kind) {
       case MacroDefinition::Kind::Invalid:
@@ -4615,7 +4617,9 @@ class Serializer::DeclSerializer : public DeclVisitor<DeclSerializer> {
       }
 
       case MacroDefinition::Kind::Expanded: {
-        llvm_unreachable("No serialization support yet");
+        expandedDef = def.getExpanded();
+        hasExpandedDefinition = 1;
+        break;
       }
     }
 
@@ -4630,13 +4634,40 @@ class Serializer::DeclSerializer : public DeclVisitor<DeclSerializer> {
                             rawAccessLevel,
                             macro->getName().getArgumentNames().size(),
                             builtinID,
+                            hasExpandedDefinition,
                             externalModuleNameID,
                             externalMacroTypeNameID,
                             nameComponentsAndDependencies);
 
     writeGenericParams(macro->getGenericParams());
     if (macro->parameterList)
       writeParameterList(macro->parameterList);
+
+    if (expandedDef) {
+      // Source text for the expanded macro definition layout.
+      uint8_t hasReplacements = !expandedDef->getReplacements().empty();
+      unsigned abbrCode =
+          S.DeclTypeAbbrCodes[ExpandedMacroDefinitionLayout::Code];
+      ExpandedMacroDefinitionLayout::emitRecord(
+          S.Out, S.ScratchRecord, abbrCode,
+          hasReplacements,
+          expandedDef->getExpansionText());
+
+      // If there are any replacements, emit a replacements record.
+      if (!hasReplacements) {
+        SmallVector<uint64_t, 3> replacements;
+        for (const auto &replacement : expandedDef->getReplacements()) {
+          replacements.push_back(replacement.startOffset);
+          replacements.push_back(replacement.endOffset);
+          replacements.push_back(replacement.parameterIndex);
+        }
+
+        unsigned abbrCode =
+            S.DeclTypeAbbrCodes[ExpandedMacroReplacementsLayout::Code];
+        ExpandedMacroReplacementsLayout::emitRecord(
+            S.Out, S.ScratchRecord, abbrCode, replacements);
+      }
+    }
   }
 
   void visitTopLevelCodeDecl(const TopLevelCodeDecl *) {
@@ -5615,6 +5646,8 @@ void Serializer::writeAllDeclsAndTypes() {
   registerDeclTypeAbbr<PrivateDiscriminatorLayout>();
   registerDeclTypeAbbr<FilenameForPrivateLayout>();
   registerDeclTypeAbbr<DeserializationSafetyLayout>();
+  registerDeclTypeAbbr<ExpandedMacroDefinitionLayout>();
+  registerDeclTypeAbbr<ExpandedMacroReplacementsLayout>();
   registerDeclTypeAbbr<MembersLayout>();
   registerDeclTypeAbbr<XRefLayout>();
 

test/Serialization/Inputs/def_macros.swift

@@ -1,5 +1,8 @@
 @freestanding(expression) public macro publicStringify<T>(_ value: T, label: String? = nil) -> (T, String) = #externalMacro(module: "MacroDefinition", type: "StringifyMacro")
 
+@freestanding(expression) public macro unlabeledStringify<T>(_ value: T) -> (T, String) = #publicStringify(value, label: "default label")
+
+
 @freestanding(expression) macro internalStringify<T>(_ value: T) -> (T, String) = #externalMacro(module: "MacroDefinition", type: "StringifyMacro")
 
 @attached(accessor) public macro myWrapper() = #externalMacro(module: "MacroDefinition", type: "MyWrapperMacro")

test/Serialization/macros.swift

@@ -14,6 +14,7 @@ import def_macros
 func test(a: Int, b: Int) {
   _ = #publicStringify(a + b)
   _ = #publicStringify(a + b, label: "hello")
+  _ = #unlabeledStringify(a + b)
 
   _ = #internalStringify(a + b)
   // expected-error@-1{{no macro named 'internalStringify'}}