[CodeCompletion] Suggest synthesized declarations from macros

* Don't invalidate the lookup cache in 'getOrCreateSynthesizedFile()'
  Adding a synthesized file itself doesn't introduce any decls. Instead,
  we should invalidate the right after the actual declrations are added
  in the file
* Remove 'SourceLookupCache::invalidate()' method. It was just used
  right before the destruction. It was just not necessary
* Include auxiliary decls in 'SourceLookupCache::lookupVisibleDecls()'
  Previously, global symbol completion didn't include decls synthesized
  by peer macros or freestanding decl macros
* Include "auxiliary decls" in visible member lookup, and visible local
  decl lookup
* Hide macro unique names

rdar://110535113

Author: rintaro

lib/AST/Decl.cpp

@@ -3827,6 +3827,10 @@ bool ValueDecl::shouldHideFromEditor() const {
       getBaseIdentifier().str().startswith("$__"))
     return true;
 
+  // Macro unique names are only intended to be used inside the expanded code.
+  if (MacroDecl::isUniqueMacroName(getBaseName()))
+    return true;
+
   return false;
 }
 

lib/AST/Module.cpp

@@ -188,9 +188,6 @@ class swift::SourceLookupCache {
   SourceLookupCache(const SourceFile &SF);
   SourceLookupCache(const ModuleDecl &Mod);
 
-  /// Throw away as much memory as possible.
-  void invalidate();
-
   void lookupValue(DeclName Name, NLKind LookupKind,
                    OptionSet<ModuleLookupFlags> Flags,
                    SmallVectorImpl<ValueDecl*> &Result);
@@ -552,6 +549,29 @@ void SourceLookupCache::lookupVisibleDecls(ImportPath::Access AccessPath,
       Consumer.foundDecl(vd, DeclVisibilityKind::VisibleAtTopLevel);
     }
   }
+
+  populateAuxiliaryDeclCache();
+  SmallVector<MissingDecl *, 4> unexpandedDecls;
+  for (auto &entry : TopLevelAuxiliaryDecls) {
+    for (auto &decl : entry.second) {
+      unexpandedDecls.append(entry.second.begin(), entry.second.end());
+    }
+  }
+
+  // Store macro expanded decls in a 'SmallSetVector' because different
+  // MissingDecls might be created by a single macro expansion. (e.g. multiple
+  // 'names' in macro role attributes). Since expansions are cached, it doesn't
+  // cause duplicated expansions, but different 'unexpandedDecl' may report the
+  // same 'ValueDecl'.
+  SmallSetVector<ValueDecl *, 4> macroExpandedDecls;
+  for (MissingDecl *unexpandedDecl : unexpandedDecls) {
+    unexpandedDecl->forEachMacroExpandedDecl([&](ValueDecl *vd) {
+      macroExpandedDecls.insert(vd);
+    });
+  }
+  for (auto *vd : macroExpandedDecls) {
+    Consumer.foundDecl(vd, DeclVisibilityKind::VisibleAtTopLevel);
+  }
 }
 
 void SourceLookupCache::lookupClassMembers(ImportPath::Access accessPath,
@@ -608,16 +628,6 @@ void SourceLookupCache::lookupClassMember(ImportPath::Access accessPath,
   results.append(iter->second.begin(), iter->second.end());
 }
 
-void SourceLookupCache::invalidate() {
-  TopLevelValues.clear();
-  ClassMembers.clear();
-  MemberCachePopulated = false;
-
-  // std::move AllVisibleValues into a temporary to destroy its contents.
-  using SameSizeSmallVector = decltype(AllVisibleValues);
-  (void)SameSizeSmallVector{std::move(AllVisibleValues)};
-}
-
 //===----------------------------------------------------------------------===//
 // Module Implementation
 //===----------------------------------------------------------------------===//
@@ -1067,9 +1077,12 @@ void SourceFile::lookupValue(DeclName name, NLKind lookupKind,
 void ModuleDecl::lookupVisibleDecls(ImportPath::Access AccessPath,
                                     VisibleDeclConsumer &Consumer,
                                     NLKind LookupKind) const {
-  if (isParsedModule(this))
-    return getSourceLookupCache().lookupVisibleDecls(
-      AccessPath, Consumer, LookupKind);
+  if (isParsedModule(this)) {
+    auto &cache = getSourceLookupCache();
+    cache.lookupVisibleDecls(AccessPath, Consumer, LookupKind);
+    assert(Cache.get() == &cache && "cache invalidated during lookup");
+    return;
+  }
 
   FORWARD(lookupVisibleDecls, (AccessPath, Consumer, LookupKind));
 }
@@ -3524,7 +3537,6 @@ void ModuleDecl::clearLookupCache() {
     return;
 
   // Abandon any current cache. We'll rebuild it on demand.
-  Cache->invalidate();
   Cache.reset();
 }
 
@@ -3944,9 +3956,6 @@ SynthesizedFileUnit &FileUnit::getOrCreateSynthesizedFile() {
       return *thisSynth;
     SynthesizedFile = new (getASTContext()) SynthesizedFileUnit(*this);
     SynthesizedFileAndKind.setPointer(SynthesizedFile);
-    // Rebuild the source lookup caches now that we have a synthesized file
-    // full of declarations to look into.
-    getParentModule()->clearLookupCache();
   }
   return *SynthesizedFile;
 }

lib/AST/NameLookup.cpp

@@ -172,7 +172,8 @@ void UsableFilteringDeclConsumer::foundDecl(ValueDecl *D,
           if (auto *contextD = DC->getAsDecl())
             tmpLoc = contextD->getStartLoc();
         }
-        if (!SM.isBeforeInBuffer(D->getLoc(), Loc))
+        auto declLoc = DC->getParentModule()->getOriginalLocation(D->getLoc()).second;
+        if (!SM.isBeforeInBuffer(declLoc, tmpLoc))
           return;
       }
 
@@ -3875,20 +3876,48 @@ void FindLocalVal::visitBraceStmt(BraceStmt *S, bool isTopLevelCode) {
       return;
   }
 
+  // Visit inner statements first before reporting local decls in the current
+  // scope.
   for (auto elem : S->getElements()) {
     // If we have a SingleValueStmtExpr, there may be local bindings in the
     // wrapped statement.
     if (auto *E = elem.dyn_cast<Expr *>()) {
       if (auto *SVE = dyn_cast<SingleValueStmtExpr>(E))
         visit(SVE->getStmt());
+      continue;
     }
-    if (auto *S = elem.dyn_cast<Stmt*>())
+
+    if (auto *S = elem.dyn_cast<Stmt*>()) {
       visit(S);
+      continue;
+    }
   }
-  for (auto elem : S->getElements()) {
-    if (auto *D = elem.dyn_cast<Decl*>()) {
+
+  auto visitDecl = [&](Decl *D) {
+    if (auto *VD = dyn_cast<ValueDecl>(D))
+      checkValueDecl(VD, DeclVisibilityKind::LocalVariable);
+    D->visitAuxiliaryDecls([&](Decl *D) {
       if (auto *VD = dyn_cast<ValueDecl>(D))
         checkValueDecl(VD, DeclVisibilityKind::LocalVariable);
+      // FIXME: Recursively call `visitDecl` to handle nested macros.
+    });
+  };
+
+  for (auto elem : S->getElements()) {
+    if (auto *E = elem.dyn_cast<Expr *>()) {
+      // 'MacroExpansionExpr' at code-item position may introduce value decls.
+      // NOTE: the expression must be type checked.
+      // FIXME: In code-completion local expressions are _not_ type checked.
+      if (auto *mee = dyn_cast<MacroExpansionExpr>(E)) {
+        if (auto *med = mee->getSubstituteDecl()) {
+          visitDecl(med);
+        }
+      }
+      continue;
+    }
+    if (auto *D = elem.dyn_cast<Decl*>()) {
+      visitDecl(D);
+      continue;
     }
   }
 }

lib/SILOptimizer/Differentiation/LinearMapInfo.cpp

@@ -117,6 +117,7 @@ LinearMapInfo::createBranchingTraceDecl(SILBasicBlock *originalBB,
     branchingTraceDecl->setAccess(AccessLevel::Internal);
   }
   file.addTopLevelDecl(branchingTraceDecl);
+  file.getParentModule()->clearLookupCache();
 
   return branchingTraceDecl;
 }

lib/Sema/LookupVisibleDecls.cpp

@@ -217,17 +217,24 @@ static void collectVisibleMemberDecls(const DeclContext *CurrDC, LookupState LS,
                                       Type BaseType,
                                       IterableDeclContext *Parent,
                                       SmallVectorImpl<ValueDecl *> &FoundDecls) {
-  for (auto Member : Parent->getMembers()) {
-    auto *VD = dyn_cast<ValueDecl>(Member);
+  auto check = [&](Decl *decl) {
+    auto *VD = dyn_cast<ValueDecl>(decl);
     if (!VD)
-      continue;
+      return;
     if (!isDeclVisibleInLookupMode(VD, LS, CurrDC))
-      continue;
+      return;
     if (!evaluateOrDefault(CurrDC->getASTContext().evaluator,
         IsDeclApplicableRequest(DeclApplicabilityOwner(CurrDC, BaseType, VD)),
                            false))
-      continue;
+      return;
     FoundDecls.push_back(VD);
+  };
+
+  for (auto Member : Parent->getMembers()) {
+    check(Member);
+    Member->visitAuxiliaryDecls([&](Decl *d) {
+      check(d);
+    });
   }
 }
 
@@ -634,16 +641,6 @@ static void synthesizeMemberDeclsForLookup(NominalTypeDecl *NTD,
                             false);
   }
 
-  // Expand peer macros.
-  for (auto *member : NTD->getMembers()) {
-    if (!ctx.evaluator.hasActiveRequest(ExpandPeerMacroRequest{member})) {
-      (void)evaluateOrDefault(
-          ctx.evaluator,
-          ExpandPeerMacroRequest{member},
-          {});
-    }
-  }
-
   synthesizePropertyWrapperVariables(NTD);
 }
 

test/IDE/complete_macros_expanded.swift

@@ -0,0 +1,96 @@
+// REQUIRES: swift_swift_parser
+
+// RUN: %empty-directory(%t)
+// RUN: mkdir -p %t/plugins
+
+//##-- Prepare the macro plugin.
+// RUN: %host-build-swift -swift-version 5 -emit-library -o %t/plugins/%target-library-name(MacroDefinition) -module-name=MacroDefinition %S/../Macros/Inputs/syntax_macro_definitions.swift -g -no-toolchain-stdlib-rpath
+
+// RUN: %target-swift-ide-test -batch-code-completion -source-filename %s -filecheck %raw-FileCheck -completion-output-dir %t -plugin-path %t/plugins -parse-as-library 
+
+@freestanding(declaration, names: named(A), named(B), named(foo), named(addOne))
+macro defineDeclsWithKnownNames() = #externalMacro(module: "MacroDefinition", type: "DefineDeclsWithKnownNamesMacro")
+
+@attached(peer, names: suffixed(_peer))
+macro PeerWithSuffix() = #externalMacro(module: "MacroDefinition", type: "PeerValueWithSuffixNameMacro")
+
+@attached(peer, names: arbitrary)
+macro PeerWithSuffixAsArbitrary() = #externalMacro(module: "MacroDefinition", type: "PeerValueWithSuffixNameMacro")
+
+@freestanding(declaration, names: arbitrary)
+macro VarValueDecl() = #externalMacro(module: "MacroDefinition", type: "VarValueMacro")
+
+
+#defineDeclsWithKnownNames
+
+@PeerWithSuffix
+func globalFunc() {}
+
+func test() {
+   #^GLOBAL^#
+// GLOBAL-DAG: Decl[Struct]/CurrModule:            A[#A#]; name=A
+// GLOBAL-DAG: Decl[Struct]/CurrModule:            B[#B#]; name=B
+// GLOBAL-DAG: Decl[GlobalVar]/CurrModule:         foo[#Int#]; name=foo
+// GLOBAL-DAG: Decl[GlobalVar]/CurrModule:         addOne[#(Int) -> Int#]; name=addOne
+// GLOBAL-DAG: Decl[FreeFunction]/CurrModule:      globalFunc()[#Void#]; name=globalFunc()
+// GLOBAL-DAG: Decl[GlobalVar]/CurrModule:         globalFunc_peer[#Int#]; name=globalFunc_peer
+}
+
+struct MyStruct {
+  @PeerWithSuffix
+  func instanceMethod() {}
+
+  @PeerWithSuffix
+  static func staticMethod() {}
+
+  @PeerWithSuffixAsArbitrary
+  func forArbitrary() {}
+
+  #defineDeclsWithKnownNames
+
+  #VarValueDecl
+}
+
+func testMemberInstance(value: MyStruct) {
+  value.#^MEMBER_INSTANCE^#
+// MEMBER_INSTANCE-DAG: Keyword[self]/CurrNominal:          self[#MyStruct#]; name=self
+// MEMBER_INSTANCE-DAG: Decl[InstanceMethod]/CurrNominal:   instanceMethod()[#Void#]; name=instanceMethod()
+// MEMBER_INSTANCE-DAG: Decl[InstanceVar]/CurrNominal:      instanceMethod_peer[#Int#]; name=instanceMethod_peer
+// MEMBER_INSTANCE-DAG: Decl[InstanceVar]/CurrNominal:      staticMethod_peer[#Int#]; name=staticMethod_peer
+// MEMBER_INSTANCE-DAG: Decl[InstanceMethod]/CurrNominal:   forArbitrary()[#Void#]; name=forArbitrary()
+// MEMBER_INSTANCE-DAG: Decl[InstanceVar]/CurrNominal:      forArbitrary_peer[#Int#]; name=forArbitrary_peer
+// MEMBER_INSTANCE-DAG: Decl[InstanceVar]/CurrNominal:      foo[#Int#]; name=foo
+// MEMBER_INSTANCE-DAG: Decl[InstanceVar]/CurrNominal:      addOne[#(Int) -> Int#]; name=addOne
+// MEMBER_INSTANCE-DAG: Decl[InstanceVar]/CurrNominal:      value[#Int#]; name=value
+// NOTE: 'staticMethod_peer' is a instance var because the macro emits the decl without 'static'
+}
+
+func testMemberStatic() {
+  MyStruct.#^MEMBER_STATIC^#
+// MEMBER_STATIC-NOT: _peer
+// MEMBER_STATIC-DAG: Keyword[self]/CurrNominal:          self[#MyStruct.Type#]; name=self
+// MEMBER_STATIC-DAG: Keyword/CurrNominal:                Type[#MyStruct.Type#]; name=Type
+// MEMBER_STATIC-DAG: Decl[Struct]/CurrNominal:           A[#MyStruct.A#]; name=A
+// MEMBER_STATIC-DAG: Decl[Struct]/CurrNominal:           B[#MyStruct.B#]; name=B
+// MEMBER_STATIC-DAG: Decl[InstanceMethod]/CurrNominal:   instanceMethod({#(self): MyStruct#})[#() -> Void#]; name=instanceMethod(:)
+// MEMBER_STATIC-DAG: Decl[StaticMethod]/CurrNominal:     staticMethod()[#Void#]; name=staticMethod()
+// MEMBER_STATIC-NOT: _peer
+}
+
+func testLocal() {
+  #defineDeclsWithKnownNames
+
+  @PeerWithSuffix func localFunc() {}
+
+  do {
+    #^LOCAL?skip=FIXME^#
+// FIXME: macros in replace function bodies are not handled correclty.
+// FIXME: decls instroduced by #defineDeclsWithKnownNames are missing.
+// LOCAL-DAG: Decl[FreeFunction]/Local:           localFunc()[#Void#]; name=localFunc()
+// LOCAL-DAG: Decl[LocalVar]/Local:               localFunc_peer[#Int#]; name=localFunc_peer
+// LOCAL-DAG: Decl[Struct]/Local:            A[#A#]; name=A
+// LOCAL-DAG: Decl[Struct]/Local:            B[#B#]; name=B
+// LOCAL-DAG: Decl[LocalVar]/Local:         foo[#Int#]; name=foo
+// LOCAL-DAG: Decl[LocalVar]/Local:         addOne[#(Int) -> Int#]; name=addOne
+  }
+}

test/Macros/Inputs/syntax_macro_definitions.swift

@@ -1626,3 +1626,16 @@ public struct InitializableMacro: ConformanceMacro, MemberMacro {
     return [requirement]
   }
 }
+
+public struct PeerValueWithSuffixNameMacro: PeerMacro {
+  public static func expansion(
+    of node: AttributeSyntax,
+    providingPeersOf declaration: some DeclSyntaxProtocol,
+    in context: some MacroExpansionContext
+  ) throws -> [DeclSyntax] {
+    guard let identified = declaration.asProtocol(IdentifiedDeclSyntax.self) else {
+      throw CustomError.message("Macro can only be applied to an identified declarations.")
+    }
+    return ["var \(raw: identified.identifier.text)_peer: Int { 1 }"]
+  }
+}