[Macros] Top-level freestanding macros

Allow freestanding macros to be used at top-level.
- Parse top-level `#...` as `MacroExpansionDecl`. When it resolves to an expression macro, expand the decl to a `MacroExpansionExpr`, and then type-check that as a normal expression.
- Add macro expansions to the source lookup cache.
- Add name lookup flags `MacroLookup` and `DisableMacroExpansion` to break dependency cycles between name lookup and macro expansion.
  - `MacroLookup` is used to prevent unqualified macro lookup from looking into local AST scopes that may contain other macro expansions.
  - `DisableMacroExpansion` is used to prevent the top-level lookup cache from eagerly expanding macros. This requires `NLOptions` to be plumbed through `lookupValue`.
- Generate AST scopes for macro expansion decls and use them to break dependency cycles, specifically to determine whether a location is within a macro declaration or top-level expansion. This work hand in hand with `DisableMacroExpansion` to prevent macro expansions within macro definitions (e.g. `#externalMacro`) from circularly depending on macro expansions to produce macro declarations for lookup.

TODOs:
- Nested expansions and name-guided expansions are not yet handled in top-level name lookup.
- Support local scope declaration macro expansions.

Author: rxwei

include/swift/AST/ASTScope.h

@@ -278,10 +278,12 @@ class ASTScopeImpl : public ASTAllocated<ASTScopeImpl> {
   static const ASTScopeImpl *findStartingScopeForLookup(SourceFile *,
                                                         const SourceLoc where);
 
-protected:
+public:
   /// Not const because may reexpand some scopes.
   ASTScopeImpl *findInnermostEnclosingScope(SourceLoc,
                                             NullablePtr<raw_ostream>);
+
+protected:
   ASTScopeImpl *findInnermostEnclosingScopeImpl(SourceLoc,
                                                 NullablePtr<raw_ostream>,
                                                 SourceManager &,
@@ -1217,6 +1219,29 @@ class MacroDeclScope final : public ASTScopeImpl {
   NullablePtr<const GenericParamList> genericParams() const override;
 };
 
+class MacroExpansionDeclScope final : public ASTScopeImpl {
+public:
+  MacroExpansionDecl *const decl;
+
+  MacroExpansionDeclScope(MacroExpansionDecl *d) : decl(d) {}
+  virtual ~MacroExpansionDeclScope() {}
+
+protected:
+  ASTScopeImpl *expandSpecifically(ScopeCreator &scopeCreator) override;
+
+private:
+  void expandAScopeThatDoesNotCreateANewInsertionPoint(ScopeCreator &);
+
+public:
+  std::string getClassName() const override;
+  SourceRange
+  getSourceRangeOfThisASTNode(bool omitAssertions = false) const override;
+
+public:
+  virtual NullablePtr<Decl> getDeclIfAny() const override { return decl; }
+  Decl *getDecl() const { return decl; }
+};
+
 class AbstractStmtScope : public ASTScopeImpl {
 public:
   SourceRange

include/swift/AST/Decl.h

@@ -109,6 +109,7 @@ namespace swift {
   class VarDecl;
   class OpaqueReturnTypeRepr;
   class Witness;
+  class MacroExpansionExpr;
 
   namespace ast_scope {
   class AbstractPatternEntryScope;
@@ -8456,13 +8457,7 @@ class MacroExpansionDecl : public Decl {
                      SourceLoc leftAngleLoc,
                      ArrayRef<TypeRepr *> genericArgs,
                      SourceLoc rightAngleLoc,
-                     ArgumentList *args)
-      : Decl(DeclKind::MacroExpansion, dc), PoundLoc(poundLoc),
-        MacroName(macro), MacroNameLoc(macroLoc),
-        LeftAngleLoc(leftAngleLoc), RightAngleLoc(rightAngleLoc),
-        GenericArgs(genericArgs), ArgList(args) {
-    Bits.MacroExpansionDecl.Discriminator = InvalidDiscriminator;
-  }
+                     ArgumentList *args);
 
   ArrayRef<TypeRepr *> getGenericArgs() const { return GenericArgs; }
 
@@ -8476,9 +8471,11 @@ class MacroExpansionDecl : public Decl {
   DeclNameLoc getMacroNameLoc() const { return MacroNameLoc; }
   DeclNameRef getMacroName() const { return MacroName; }
   ArgumentList *getArgs() const { return ArgList; }
-  ArrayRef<Decl *> getRewritten() const;
+  void setArgs(ArgumentList *args) { ArgList = args; }
+  ArrayRef<ASTNode> getRewritten() const;
   ConcreteDeclRef getMacroRef() const { return macroRef; }
   void setMacroRef(ConcreteDeclRef ref) { macroRef = ref; }
+  MacroExpansionExpr *createExpr() const;
 
   /// Returns a discriminator which determines this macro expansion's index
   /// in the sequence of macro expansions within the current function.

include/swift/AST/DeclContext.h

@@ -832,6 +832,13 @@ class IterableDeclContext {
   /// The resulting list of members will be stable across translation units.
   ArrayRef<Decl *> getAllMembers() const;
 
+  /// Get all of the members within this context, including any
+  /// implicitly-synthesized members and macro expansions. All macro expansions
+  /// will be recursively flattened.
+  ///
+  /// The resulting list of members will be stable across translation units.
+  ArrayRef<Decl *> getExpandedAllMembers() const;
+
   /// Retrieve the set of members in this context without loading any from the
   /// associated lazy loader; this should only be used as part of implementing
   /// abstractions on top of member loading, such as a name lookup table.

include/swift/AST/FileUnit.h

@@ -64,9 +64,14 @@ class FileUnit : public DeclContext, public ASTAllocated<FileUnit> {
   /// within this file.
   ///
   /// This does a simple local lookup, not recursively looking through imports.
-  virtual void lookupValue(DeclName name, NLKind lookupKind,
+  virtual void lookupValue(DeclName name, NLKind lookupKind, NLOptions options,
                            SmallVectorImpl<ValueDecl*> &result) const = 0;
 
+  void lookupValue(DeclName name, NLKind lookupKind,
+                   SmallVectorImpl<ValueDecl*> &result) const {
+    lookupValue(name, lookupKind, getDefaultNLOptions(lookupKind), result);
+  }
+
   /// Look up a local type declaration by its mangled name.
   ///
   /// This does a simple local lookup, not recursively looking through imports.
@@ -210,6 +215,13 @@ class FileUnit : public DeclContext, public ASTAllocated<FileUnit> {
   /// The order of the results is not guaranteed to be meaningful.
   virtual void getTopLevelDecls(SmallVectorImpl<Decl*> &results) const {}
 
+  /// Finds all top-level decls in this file with all macros recursively
+  /// expanded.
+  ///
+  /// This does a simple local lookup, not recursively looking through imports.
+  /// The order of the results is not guaranteed to be meaningful.
+  void getExpandedTopLevelDecls(SmallVectorImpl<Decl*> &results) const;
+
   virtual void
   getExportedPrespecializations(SmallVectorImpl<Decl *> &results) const {}
 
@@ -369,7 +381,7 @@ class BuiltinUnit final : public FileUnit {
 public:
   explicit BuiltinUnit(ModuleDecl &M);
 
-  virtual void lookupValue(DeclName name, NLKind lookupKind,
+  virtual void lookupValue(DeclName name, NLKind lookupKind, NLOptions options,
                            SmallVectorImpl<ValueDecl*> &result) const override;
 
   /// Find all Objective-C methods with the given selector.

include/swift/AST/LookupKinds.h

@@ -53,6 +53,9 @@ enum NLOptions : unsigned {
   // Include @usableFromInline and @inlinable
   NL_IncludeUsableFromInline = 1 << 6,
 
+  /// Don't expand macros.
+  NL_DisableMacroExpansions = 1 << 7,
+
   /// The default set of options used for qualified name lookup.
   ///
   /// FIXME: Eventually, add NL_ProtocolMembers to this, once all of the
@@ -79,6 +82,13 @@ static inline NLOptions operator~(NLOptions value) {
   return NLOptions(~(unsigned)value);
 }
 
+static inline NLOptions getDefaultNLOptions(NLKind kind) {
+  switch (kind) {
+  case NLKind::QualifiedLookup: return NL_QualifiedDefault;
+  case NLKind::UnqualifiedLookup: return NL_UnqualifiedDefault;
+  }
+}
+
 void simple_display(llvm::raw_ostream &out, NLOptions options);
 
 } // end namespace swift

include/swift/AST/Module.h

@@ -651,9 +651,19 @@ class ModuleDecl
   /// within the current module.
   ///
   /// This does a simple local lookup, not recursively looking through imports.
-  void lookupValue(DeclName Name, NLKind LookupKind,
+  void lookupValue(DeclName Name, NLKind LookupKind, NLOptions Options,
                    SmallVectorImpl<ValueDecl*> &Result) const;
 
+  /// Look up a (possibly overloaded) value set at top-level scope
+  /// (but with the specified access path, which may come from an import decl)
+  /// within the current module.
+  ///
+  /// This does a simple local lookup, not recursively looking through imports.
+  void lookupValue(DeclName Name, NLKind LookupKind,
+                   SmallVectorImpl<ValueDecl*> &Result) const {
+    lookupValue(Name, LookupKind, getDefaultNLOptions(LookupKind), Result);
+  }
+
   /// Look up a local type declaration by its mangled name.
   ///
   /// This does a simple local lookup, not recursively looking through imports.

include/swift/AST/NameLookup.h

@@ -239,6 +239,10 @@ enum class UnqualifiedLookupFlags {
   // This lookup should include results that are @inlinable or
   // @usableFromInline.
   IncludeUsableFromInline = 1 << 5,
+  // This lookup should not expand macros.
+  DisableMacroExpansions  = 1 << 6,
+  // This lookup should only return macros.
+  MacroLookup             = 1 << 7,
 };
 
 using UnqualifiedLookupOptions = OptionSet<UnqualifiedLookupFlags>;
@@ -794,6 +798,11 @@ class ASTScope : public ASTAllocated<ASTScope> {
   static std::pair<CaseStmt *, CaseStmt *>
   lookupFallthroughSourceAndDest(SourceFile *sourceFile, SourceLoc loc);
 
+  /// Returns true if the given location is within a \c MacroDecl or a top-level
+  /// \c MacroExpansionDecl.
+  static bool isLocWithinMacroDeclOrTopLevelMacroExpansionDeclScope(
+      SourceFile *sf, SourceLoc loc);
+
   SWIFT_DEBUG_DUMP;
   void print(llvm::raw_ostream &) const;
   void dumpOneScopeMapLocation(std::pair<unsigned, unsigned>);

include/swift/AST/SourceFile.h

@@ -427,7 +427,7 @@ class SourceFile final : public FileUnit {
   void cacheVisibleDecls(SmallVectorImpl<ValueDecl *> &&globals) const;
   const SmallVectorImpl<ValueDecl *> &getCachedVisibleDecls() const;
 
-  virtual void lookupValue(DeclName name, NLKind lookupKind,
+  virtual void lookupValue(DeclName name, NLKind lookupKind, NLOptions options,
                            SmallVectorImpl<ValueDecl*> &result) const override;
 
   virtual void lookupVisibleDecls(ImportPath::Access accessPath,

include/swift/AST/SynthesizedFileUnit.h

@@ -43,7 +43,7 @@ class SynthesizedFileUnit final : public FileUnit {
   /// Add a synthesized top-level declaration.
   void addTopLevelDecl(Decl *D) { TopLevelDecls.push_back(D); }
 
-  virtual void lookupValue(DeclName name, NLKind lookupKind,
+  virtual void lookupValue(DeclName name, NLKind lookupKind, NLOptions options,
                            SmallVectorImpl<ValueDecl *> &result) const override;
 
   void lookupObjCMethods(

include/swift/AST/TypeCheckRequests.h

@@ -1813,6 +1813,24 @@ class AllMembersRequest :
   bool isCached() const { return true; }
 };
 
+class ExpandedAllMembersRequest :
+    public SimpleRequest<ExpandedAllMembersRequest,
+                         ArrayRef<Decl *>(IterableDeclContext *),
+                         RequestFlags::Cached> {
+public:
+  using SimpleRequest::SimpleRequest;
+
+private:
+  friend SimpleRequest;
+
+  // Evaluation.
+  ArrayRef<Decl *>
+  evaluate(Evaluator &evaluator, IterableDeclContext *idc) const;
+
+public:
+  bool isCached() const { return true; }
+};
+
 class IsImplicitlyUnwrappedOptionalRequest :
     public SimpleRequest<IsImplicitlyUnwrappedOptionalRequest,
                          bool(ValueDecl *),
@@ -3813,16 +3831,16 @@ class MacroDefinitionRequest
 /// Find the definition of a given macro.
 class ExpandMacroExpansionDeclRequest
     : public SimpleRequest<ExpandMacroExpansionDeclRequest,
-                           ArrayRef<Decl *>(MacroExpansionDecl *),
+                           ArrayRef<ASTNode>(MacroExpansionDecl *),
                            RequestFlags::Cached> {
 public:
   using SimpleRequest::SimpleRequest;
 
 private:
   friend SimpleRequest;
 
-  ArrayRef<Decl *> evaluate(Evaluator &evaluator,
-                            MacroExpansionDecl *med) const;
+  ArrayRef<ASTNode>
+  evaluate(Evaluator &evaluator, MacroExpansionDecl *med) const;
 
 public:
   bool isCached() const { return true; }

include/swift/AST/TypeCheckerTypeIDZone.def

@@ -83,6 +83,8 @@ SWIFT_REQUEST(TypeChecker, ABIMembersRequest,
               ArrayRef<Decl *>(IterableDeclContext *), Cached, NoLocationInfo)
 SWIFT_REQUEST(TypeChecker, AllMembersRequest,
               ArrayRef<Decl *>(IterableDeclContext *), Cached, NoLocationInfo)
+SWIFT_REQUEST(TypeChecker, ExpandedAllMembersRequest,
+              ArrayRef<Decl *>(IterableDeclContext *), Cached, NoLocationInfo)
 SWIFT_REQUEST(TypeChecker, SpecializeAttrTargetDeclRequest,
               ValueDecl *(const ValueDecl *, SpecializeAttr *),
               Cached, NoLocationInfo)
@@ -429,7 +431,7 @@ SWIFT_REQUEST(TypeChecker, ExternalMacroDefinitionRequest,
               ExternalMacroDefinition(ASTContext *, Identifier, Identifier),
               Cached, NoLocationInfo)
 SWIFT_REQUEST(TypeChecker, ExpandMacroExpansionDeclRequest,
-              ArrayRef<Decl *>(MacroExpansionDecl *),
+              ArrayRef<ASTNode> (MacroExpansionDecl *),
               Cached, NoLocationInfo)
 SWIFT_REQUEST(TypeChecker, ExpandMemberAttributeMacros,
               ArrayRef<unsigned>(Decl *),

include/swift/AST/TypeMemberVisitor.h

@@ -81,8 +81,9 @@ class TypeMemberVisitor : public DeclVisitor<ImplClass, RetTy> {
 
   /// Visit expanded macros.
   void visitMacroExpansionDecl(MacroExpansionDecl *D) {
-    for (auto *decl : D->getRewritten())
-      asImpl().visit(decl);
+    for (auto node : D->getRewritten())
+      if (auto *decl = node.dyn_cast<Decl *>())
+        asImpl().visit(decl);
   }
 };
 

include/swift/ClangImporter/ClangModule.h

@@ -65,7 +65,7 @@ class ClangModuleUnit final : public LoadedFile {
 
   virtual bool isSystemModule() const override;
 
-  virtual void lookupValue(DeclName name, NLKind lookupKind,
+  virtual void lookupValue(DeclName name, NLKind lookupKind, NLOptions options,
                            SmallVectorImpl<ValueDecl*> &results) const override;
 
   virtual TypeDecl *

include/swift/Serialization/SerializedModuleLoader.h

@@ -359,7 +359,7 @@ class SerializedASTFile final : public LoadedFile {
 
   virtual bool isSystemModule() const override;
 
-  virtual void lookupValue(DeclName name, NLKind lookupKind,
+  virtual void lookupValue(DeclName name, NLKind lookupKind, NLOptions options,
                            SmallVectorImpl<ValueDecl*> &results) const override;
 
   virtual StringRef

lib/AST/ASTDumper.cpp

@@ -1322,12 +1322,12 @@ namespace {
     void visitMacroExpansionDecl(MacroExpansionDecl *MED) {
       printCommon(MED, "macro_expansion_decl ");
       OS << MED->getMacroName();
-      if (MED->getArgs()) {
-        OS << '\n';
-        OS.indent(Indent + 2);
-        printArgumentList(OS, MED->getArgs(), Indent,
-                          [&](Expr *E) { printRec(E); });
-      }
+      OS << '\n';
+      printArgumentList(OS, MED->getArgs(), Indent,
+                        [&](Expr *E) { printRec(E); });
+      auto rewritten = MED->getRewritten();
+      if (!rewritten.empty())
+        printASTNodes(rewritten, MED->getASTContext(), "rewritten");
       PrintWithColorRAII(OS, ParenthesisColor) << ')';
     }
   };
@@ -3045,7 +3045,7 @@ class PrintExpr : public ExprVisitor<PrintExpr> {
       printArgumentList(E->getArgs());
     }
     if (auto rewritten = E->getRewritten()) {
-      OS << '\n';
+      OS << " rewritten=\n";
       printRec(rewritten);
     }
     PrintWithColorRAII(OS, ParenthesisColor) << ')';

lib/AST/ASTPrinter.cpp

@@ -4608,20 +4608,18 @@ void PrintAST::visitMacroDecl(MacroDecl *decl) {
 
 void PrintAST::visitMacroExpansionDecl(MacroExpansionDecl *decl) {
   Printer << '#' << decl->getMacroName();
-  if (decl->getArgs()) {
-    Printer << '(';
-    auto args = decl->getArgs()->getOriginalArgs();
-    bool isFirst = true;
-    // FIXME: handle trailing closures.
-    for (auto arg : *args) {
-      if (!isFirst) {
-        Printer << ", ";
-      }
-      printArgument(arg);
-      isFirst = false;
+  Printer << '(';
+  auto args = decl->getArgs()->getOriginalArgs();
+  bool isFirst = true;
+  // FIXME: handle trailing closures.
+  for (auto arg : *args) {
+    if (!isFirst) {
+      Printer << ", ";
     }
-    Printer << ')';
+    printArgument(arg);
+    isFirst = false;
   }
+  Printer << ')';
 }
 
 void PrintAST::visitIntegerLiteralExpr(IntegerLiteralExpr *expr) {