[Macros] Start expanding accessor macros attached to a storage declaration.

Accessor macros are attached macros (written with attribute syntax)
that can generate accessors for a property or subscript. Recognize
custom attributes that are accessor macros when written on a storage
declaration, and expand those macros.

This is very much a work in progress, and the result of the expansion
isn't yet parsed or wired into the AST.

Author: DougGregor

include/swift/Basic/SourceManager.h

@@ -23,6 +23,7 @@
 
 namespace swift {
 
+class CustomAttr;
 class DeclContext;
 
 /// Augments a buffer that was created specifically to hold generated source
@@ -59,6 +60,9 @@ class GeneratedSourceInfo {
 
   /// The declaration context in which this buffer logically resides.
   DeclContext *declContext;
+
+  /// The custom attribute for an attached macro.
+  CustomAttr *attachedMacroCustomAttr = nullptr;
 };
 
 /// This class manages and owns source buffers.

lib/ASTGen/Sources/ASTGen/Macros.swift

@@ -215,3 +215,138 @@ func evaluateMacro(
 
   return 0
 }
+
+/// Retrieve a syntax node in the given source file, with the given type.
+private func findSyntaxNodeInSourceFile<Node: SyntaxProtocol>(
+  sourceFilePtr: UnsafeRawPointer,
+  sourceLocationPtr: UnsafePointer<UInt8>?,
+  type: Node.Type
+) -> Node? {
+  guard let sourceLocationPtr = sourceLocationPtr else {
+    return nil
+  }
+
+  let sourceFilePtr = sourceFilePtr.bindMemory(
+    to: ExportedSourceFile.self, capacity: 1
+  )
+
+  // Find the offset.
+  let buffer = sourceFilePtr.pointee.buffer
+  let offset = sourceLocationPtr - buffer.baseAddress!
+  if offset < 0 || offset >= buffer.count {
+    print("source location isn't inside this buffer")
+    return nil
+  }
+
+  // Find the token at that offset.
+  let sf = sourceFilePtr.pointee.syntax
+  guard let token = sf.token(at: AbsolutePosition(utf8Offset: offset)) else {
+    print("couldn't find token at offset \(offset)")
+    return nil
+  }
+
+  // Dig out its parent.
+  guard let parentSyntax = token.parent else {
+    print("not on a macro expansion node: \(token.recursiveDescription)")
+    return nil
+  }
+
+  return parentSyntax.as(type)
+}
+
+@_cdecl("swift_ASTGen_expandAttachedMacro")
+@usableFromInline
+func expandAttachedMacro(
+  diagEnginePtr: UnsafeMutablePointer<UInt8>,
+  macroPtr: UnsafeRawPointer,
+  customAttrSourceFilePtr: UnsafeRawPointer,
+  customAttrSourceLocPointer: UnsafePointer<UInt8>?,
+  declarationSourceFilePtr: UnsafeRawPointer,
+  attachedTo declarationSourceLocPointer: UnsafePointer<UInt8>?,
+  expandedSourcePointer: UnsafeMutablePointer<UnsafePointer<UInt8>?>,
+  expandedSourceLength: UnsafeMutablePointer<Int>
+) -> Int {
+  // We didn't expand anything so far.
+  expandedSourcePointer.pointee = nil
+  expandedSourceLength.pointee = 0
+
+  // Dig out the custom attribute for the attached macro declarations.
+  guard let customAttrNode = findSyntaxNodeInSourceFile(
+    sourceFilePtr: customAttrSourceFilePtr,
+    sourceLocationPtr: customAttrSourceLocPointer,
+    type: AttributeSyntax.self
+  ) else {
+    return 1
+  }
+
+  // Dig out the node for the declaration to which the custom attribute is
+  // attached.
+  guard let declarationNode = findSyntaxNodeInSourceFile(
+    sourceFilePtr: declarationSourceFilePtr,
+    sourceLocationPtr: declarationSourceLocPointer,
+    type: DeclSyntax.self
+  ) else {
+    return 1
+  }
+
+  // Get the macro.
+  let macroPtr = macroPtr.bindMemory(to: ExportedMacro.self, capacity: 1)
+  let macro = macroPtr.pointee.macro
+
+  // FIXME: Which source file? I don't know! This should go.
+  let declarationSourceFilePtr = declarationSourceFilePtr.bindMemory(
+    to: ExportedSourceFile.self, capacity: 1
+  )
+
+  var context = MacroExpansionContext(
+    moduleName: declarationSourceFilePtr.pointee.moduleName,
+    fileName: declarationSourceFilePtr.pointee.fileName.withoutPath()
+  )
+
+  var evaluatedSyntaxStr: String
+  do {
+    switch macro {
+    case let attachedMacro as AccessorDeclarationMacro.Type:
+      let accessors = try attachedMacro.expansion(
+        of: customAttrNode, attachedTo: declarationNode, in: &context
+      )
+
+      // Form a buffer of accessor declarations to return to the caller.
+      evaluatedSyntaxStr = accessors.map {
+        $0.withoutTrivia().description
+      }.joined(separator: "\n\n")
+
+    default:
+      print("\(macroPtr) does not conform to any known attached macro protocol")
+      return 1
+    }
+  } catch {
+    // Record the error
+    // FIXME: Need to decide where to diagnose the error:
+    context.diagnose(
+      Diagnostic(
+        node: Syntax(declarationNode),
+        message: ThrownErrorDiagnostic(message: String(describing: error))
+      )
+    )
+
+    return 1
+  }
+
+  // FIXME: Emit diagnostics, but how do we figure out which source file to
+  // use?
+
+  // Form the result buffer for our caller.
+  evaluatedSyntaxStr.withUTF8 { utf8 in
+    let evaluatedResultPtr = UnsafeMutablePointer<UInt8>.allocate(capacity: utf8.count + 1)
+    if let baseAddress = utf8.baseAddress {
+      evaluatedResultPtr.initialize(from: baseAddress, count: utf8.count)
+    }
+    evaluatedResultPtr[utf8.count] = 0
+
+    expandedSourcePointer.pointee = UnsafePointer(evaluatedResultPtr)
+    expandedSourceLength.pointee = utf8.count
+  }
+
+  return 0
+}

lib/Sema/TypeCheckMacros.cpp

@@ -42,6 +42,16 @@ extern "C" ptrdiff_t swift_ASTGen_evaluateMacro(
     const void *sourceLocation,
     const char **evaluatedSource, ptrdiff_t *evaluatedSourceLength);
 
+extern "C" ptrdiff_t swift_ASTGen_expandAttachedMacro(
+    void *diagEngine, void *macro,
+    void *customAttrSourceFile,
+    const void *customAttrSourceLocation,
+    void *declarationSourceFile,
+    const void *declarationSourceLocation,
+    const char **evaluatedSource,
+    ptrdiff_t *evaluatedSourceLength
+);
+
 /// Produce the mangled name for the nominal type descriptor of a type
 /// referenced by its module and type name.
 static std::string mangledNameForTypeMetadataAccessor(
@@ -662,3 +672,193 @@ bool swift::expandFreestandingDeclarationMacro(
   }
   return true;
 }
+
+void swift::expandAccessors(
+    AbstractStorageDecl *storage, CustomAttr *attr, MacroDecl *macro
+) {
+  auto *dc = storage->getInnermostDeclContext();
+  ASTContext &ctx = dc->getASTContext();
+  SourceManager &sourceMgr = ctx.SourceMgr;
+
+  auto moduleDecl = dc->getParentModule();
+
+  auto attrSourceFile =
+    moduleDecl->getSourceFileContainingLocation(attr->AtLoc);
+  if (!attrSourceFile)
+    return;
+
+  auto declSourceFile =
+      moduleDecl->getSourceFileContainingLocation(storage->getStartLoc());
+  if (!declSourceFile)
+    return;
+
+  // Evaluate the macro.
+  NullTerminatedStringRef evaluatedSource;
+
+  if (isFromExpansionOfMacro(attrSourceFile, macro) ||
+      isFromExpansionOfMacro(declSourceFile, macro)) {
+    storage->diagnose(diag::macro_recursive, macro->getName());
+    return;
+  }
+
+  auto macroDef = macro->getDefinition();
+  switch (macroDef.kind) {
+  case MacroDefinition::Kind::Undefined:
+  case MacroDefinition::Kind::Invalid:
+    // Already diagnosed as an error elsewhere.
+    return;
+
+  case MacroDefinition::Kind::Builtin: {
+    switch (macroDef.getBuiltinKind()) {
+    case BuiltinMacroKind::ExternalMacro:
+      // FIXME: Error here.
+      return;
+    }
+  }
+
+  case MacroDefinition::Kind::External: {
+    // Retrieve the external definition of the macro.
+    auto external = macroDef.getExternalMacro();
+    ExternalMacroDefinitionRequest request{
+        &ctx, external.moduleName, external.macroTypeName
+    };
+    auto externalDef = evaluateOrDefault(
+        ctx.evaluator, request, ExternalMacroDefinition()
+    );
+    if (!externalDef.opaqueHandle) {
+      storage->diagnose(diag::external_macro_not_found,
+                        external.moduleName.str(),
+                        external.macroTypeName.str(),
+                        macro->getName()
+      );
+      macro->diagnose(diag::decl_declared_here, macro->getName());
+      return;
+    }
+
+    // Make sure macros are enabled before we expand.
+    if (!ctx.LangOpts.hasFeature(Feature::Macros)) {
+      storage->diagnose(diag::macro_experimental);
+      return;
+    }
+
+#if SWIFT_SWIFT_PARSER
+    PrettyStackTraceDecl debugStack("expanding accessor macro", storage);
+
+    auto astGenAttrSourceFile = attrSourceFile->exportedSourceFile;
+    if (!astGenAttrSourceFile)
+      return;
+
+    auto astGenDeclSourceFile = declSourceFile->exportedSourceFile;
+    if (!astGenDeclSourceFile)
+      return;
+
+    Decl *searchDecl = storage;
+    if (auto var = dyn_cast<VarDecl>(storage))
+      searchDecl = var->getParentPatternBinding();
+
+    const char *evaluatedSourceAddress;
+    ptrdiff_t evaluatedSourceLength;
+    swift_ASTGen_expandAttachedMacro(
+        &ctx.Diags,
+        externalDef.opaqueHandle,
+        astGenAttrSourceFile, attr->AtLoc.getOpaquePointerValue(),
+        astGenDeclSourceFile, searchDecl->getStartLoc().getOpaquePointerValue(),
+        &evaluatedSourceAddress, &evaluatedSourceLength);
+    if (!evaluatedSourceAddress)
+      return;
+    evaluatedSource = NullTerminatedStringRef(evaluatedSourceAddress,
+                                              (size_t)evaluatedSourceLength);
+    break;
+#else
+    med->diagnose(diag::macro_unsupported);
+    return false;
+#endif
+  }
+  }
+
+  // Figure out a reasonable name for the macro expansion buffer.
+  std::string bufferName;
+  {
+    llvm::raw_string_ostream out(bufferName);
+
+    out << "macro:" << storage->getName()
+        << "@" << macro->getName().getBaseName();
+    if (auto bufferID = declSourceFile->getBufferID()) {
+      unsigned startLine, startColumn;
+      std::tie(startLine, startColumn) =
+          sourceMgr.getLineAndColumnInBuffer(storage->getStartLoc(), *bufferID);
+
+      SourceLoc endLoc =
+          Lexer::getLocForEndOfToken(sourceMgr, storage->getEndLoc());
+      unsigned endLine, endColumn;
+      std::tie(endLine, endColumn) =
+          sourceMgr.getLineAndColumnInBuffer(endLoc, *bufferID);
+
+      out << ":" << sourceMgr.getIdentifierForBuffer(*bufferID) << ":"
+          << startLine << ":" << startColumn
+          << "-" << endLine << ":" << endColumn;
+    }
+  }
+
+  // Dump macro expansions to standard output, if requested.
+  if (ctx.LangOpts.DumpMacroExpansions) {
+    llvm::errs() << bufferName
+                 << "\n------------------------------\n"
+                 << evaluatedSource
+                 << "\n------------------------------\n";
+  }
+
+  // Create a new source buffer with the contents of the expanded macro.
+  auto macroBuffer =
+      llvm::MemoryBuffer::getMemBufferCopy(evaluatedSource, bufferName);
+  unsigned macroBufferID = sourceMgr.addNewSourceBuffer(std::move(macroBuffer));
+  auto macroBufferRange = sourceMgr.getRangeForBuffer(macroBufferID);
+  GeneratedSourceInfo sourceInfo{
+      GeneratedSourceInfo::MacroExpansion,
+      storage->getEndLoc(),
+      SourceRange(macroBufferRange.getStart(), macroBufferRange.getEnd()),
+      ASTNode(storage).getOpaqueValue(),
+      dc,
+      attr
+  };
+  sourceMgr.setGeneratedSourceInfo(macroBufferID, sourceInfo);
+  free((void*)evaluatedSource.data());
+
+  // Create a source file to hold the macro buffer. This is automatically
+  // registered with the enclosing module.
+  auto macroSourceFile = new (ctx) SourceFile(
+      *dc->getParentModule(), SourceFileKind::MacroExpansion, macroBufferID,
+      /*parsingOpts=*/{}, /*isPrimary=*/false);
+
+  PrettyStackTraceDecl debugStack(
+      "type checking expanded declaration macro", storage);
+
+  // FIXME: getTopLevelItems() is going to have to figure out how to parse
+  // a sequence of accessor declarations. Specifically, we will have to
+  // encode enough information in the GeneratedSourceInfo to know that
+  // the "top level" here is really a set of accessor declarations, so we
+  // can parse those. Both parsers will need to do it this way. Alternatively,
+  // we can put some extra braces around things and use parseGetSet, but that
+  // doesn't feel quite right, because we might eventually allow additional
+  // accessors to be inserted for properties that already have accesssors.
+  // parseTopLevelItems is where things get interesting... hmmm...
+
+
+#if false
+  // Retrieve the parsed declarations from the list of top-level items.
+  auto topLevelItems = macroSourceFile->getTopLevelItems();
+  for (auto item : topLevelItems) {
+    auto *decl = item.dyn_cast<Decl *>();
+    if (!decl) {
+      ctx.Diags.diagnose(
+          macroBufferRange.getStart(), diag::expected_macro_expansion_decls);
+      return;
+    }
+    decl->setDeclContext(dc);
+    TypeChecker::typeCheckDecl(decl);
+    results.push_back(decl);
+  }
+
+#endif
+  return;
+}

lib/Sema/TypeCheckMacros.h

@@ -21,7 +21,9 @@
 
 namespace swift {
 
+class CustomAttr;
 class Expr;
+class MacroDecl;
 class MacroExpansionDecl;
 class TypeRepr;
 
@@ -40,6 +42,12 @@ Expr *expandMacroExpr(
 bool expandFreestandingDeclarationMacro(
     MacroExpansionDecl *med, SmallVectorImpl<Decl *> &results);
 
+/// Expand the accessors for the given storage declaration based on the
+/// custom attribute that references the given macro.
+void expandAccessors(
+    AbstractStorageDecl *storage, CustomAttr *attr, MacroDecl *macro
+);
+
 } // end namespace swift
 
 #endif /* SWIFT_SEMA_TYPECHECKMACROS_H */