Merge pull request #78335 from rintaro/revert-revert-77140-swift-lexical-lookup-validation

Reapply "[SwiftLexicalLookup] New unqualified lookup implementation validation"

Author: rintaro

include/swift/AST/ASTBridging.h

@@ -101,6 +101,21 @@ struct BridgedLocatedIdentifier {
   BridgedSourceLoc NameLoc;
 };
 
+struct BridgedConsumedLookupResult {
+  SWIFT_NAME("name")
+  BridgedIdentifier Name;
+
+  SWIFT_NAME("nameLoc")
+  BridgedSourceLoc NameLoc;
+
+  SWIFT_NAME("flag")
+  SwiftInt Flag;
+
+  BRIDGED_INLINE BridgedConsumedLookupResult(swift::Identifier name,
+                                             swift::SourceLoc sourceLoc,
+                                             SwiftInt flag);
+};
+
 class BridgedDeclBaseName {
   BridgedIdentifier Ident;
 

include/swift/AST/ASTBridgingImpl.h

@@ -55,6 +55,15 @@ swift::DeclBaseName BridgedDeclBaseName::unbridged() const {
   return swift::DeclBaseName(Ident.unbridged());
 }
 
+//===----------------------------------------------------------------------===//
+// MARK: BridgedDeclBaseName
+//===----------------------------------------------------------------------===//
+
+BridgedConsumedLookupResult::BridgedConsumedLookupResult(
+    swift::Identifier name, swift::SourceLoc sourceLoc, SwiftInt flag)
+    : Name(BridgedIdentifier(name)), NameLoc(BridgedSourceLoc(sourceLoc)),
+      Flag(flag) {}
+
 //===----------------------------------------------------------------------===//
 // MARK: BridgedDeclNameRef
 //===----------------------------------------------------------------------===//

include/swift/AST/DiagnosticsCommon.def

@@ -236,6 +236,13 @@ NOTE(in_macro_expansion,none,
 ERROR(macro_experimental,none,
       "%0 macros are an experimental feature that is not enabled %select{|(%1)}1",
       (StringRef, StringRef))
+      
+//------------------------------------------------------------------------------
+// MARK: lexical lookup diagnostics
+//------------------------------------------------------------------------------
+      
+ERROR(lookup_outputs_dont_match,none,
+"Unqualified lookup output from ASTScope and SwiftLexicalLookup don't match", ())
 
 //------------------------------------------------------------------------------
 // MARK: bridged diagnostics

include/swift/Basic/Features.def

@@ -300,6 +300,10 @@ EXPERIMENTAL_FEATURE(ParserRoundTrip, false)
 /// Swift parser.
 EXPERIMENTAL_FEATURE(ParserValidation, false)
 
+/// Whether to perform validation of the unqualified lookup produced by
+/// ASTScope and SwiftLexicalLookup
+EXPERIMENTAL_FEATURE(UnqualifiedLookupValidation, false)
+
 /// Enables implicit some while also enabling existential `any`
 EXPERIMENTAL_FEATURE(ImplicitSome, false)
 

include/swift/Bridging/ASTGen.h

@@ -98,6 +98,13 @@ intptr_t swift_ASTGen_configuredRegions(
 void swift_ASTGen_freeConfiguredRegions(
     BridgedIfConfigClauseRangeInfo *_Nullable regions, intptr_t numRegions);
 
+bool swift_ASTGen_validateUnqualifiedLookup(
+    void *_Nonnull sourceFile,
+    BridgedASTContext astContext,
+    BridgedSourceLoc sourceLoc,
+    bool finishInSequentialScope,
+    BridgedArrayRef astScopeResultRef);
+
 size_t
 swift_ASTGen_virtualFiles(void *_Nonnull sourceFile,
                           BridgedVirtualFile *_Nullable *_Nonnull virtualFiles);

lib/AST/ASTScope.cpp

@@ -17,6 +17,7 @@
 
 #include "swift/AST/ASTContext.h"
 #include "swift/AST/ASTWalker.h"
+#include "swift/Bridging/ASTGen.h"
 #include "swift/AST/Decl.h"
 #include "swift/AST/Expr.h"
 #include "swift/AST/Initializer.h"
@@ -39,6 +40,104 @@ using namespace ast_scope;
 
 #pragma mark ASTScope
 
+class LoggingASTScopeDeclConsumer
+    : public namelookup::AbstractASTScopeDeclConsumer {
+private:
+  const int shouldLookInMembers = 0b10;
+  namelookup::AbstractASTScopeDeclConsumer *originalConsumer;
+
+public:
+  mutable SmallVector<BridgedConsumedLookupResult> recordedElements;
+
+  LoggingASTScopeDeclConsumer(
+      namelookup::AbstractASTScopeDeclConsumer *consumer)
+      : originalConsumer(consumer) {}
+
+  ~LoggingASTScopeDeclConsumer() = default;
+
+  /// Called for every ValueDecl visible from the lookup.
+  ///
+  /// Takes an array in order to batch the consumption before setting
+  /// IndexOfFirstOuterResult when necessary.
+  ///
+  /// Additionally, each name is logged to `recordedElements` and
+  /// can be later used in validation of `SwiftLexicalLookup` result.
+  ///
+  /// \param baseDC either a type context or the local context of a
+  /// `self` parameter declaration. See LookupResult for a discussion
+  /// of type -vs- instance lookup results.
+  ///
+  /// \return true if the lookup should be stopped at this point.
+  bool consume(ArrayRef<ValueDecl *> values,
+               NullablePtr<DeclContext> baseDC = nullptr) override {
+    bool endOfLookup = originalConsumer->consume(values, baseDC);
+
+    for (auto value : values) {
+      if (auto sourceLoc = value->getLoc()) {
+        recordedElements.push_back(BridgedConsumedLookupResult(
+            value->getBaseIdentifier(), sourceLoc, endOfLookup));
+      } else {
+        // If sourceLoc is unavailable, use location of it's parent.
+        recordedElements.push_back(BridgedConsumedLookupResult(
+            value->getBaseIdentifier(),
+            value->getDeclContext()->getAsDecl()->getLoc(), endOfLookup));
+      }
+    }
+
+    return endOfLookup;
+  };
+
+  /// Look for members of a nominal type or extension scope.
+  ///
+  /// Each call is recorded in `recordedElements` with a special flag set.
+  /// It can be later used in validation of `SwiftLexicalLookup` result.
+  ///
+  /// \return true if the lookup should be stopped at this point.
+  bool lookInMembers(const DeclContext *scopeDC) const override {
+    bool endOfLookup = originalConsumer->lookInMembers(scopeDC);
+
+    if (auto *extDecl = dyn_cast<ExtensionDecl>(scopeDC)) {
+      recordedElements.push_back(BridgedConsumedLookupResult(
+          Identifier(), extDecl->getExtendedTypeRepr()->getLoc(),
+          shouldLookInMembers + endOfLookup));
+    } else {
+      recordedElements.push_back(BridgedConsumedLookupResult(
+          scopeDC->getSelfNominalTypeDecl()->getBaseIdentifier(),
+          scopeDC->getAsDecl()->getLoc(), shouldLookInMembers + endOfLookup));
+    }
+
+    return endOfLookup;
+  };
+
+  /// Called for local VarDecls that might not yet be in scope.
+  ///
+  /// Note that the set of VarDecls visited here are going to be a
+  /// superset of those visited in consume().
+  bool consumePossiblyNotInScope(ArrayRef<VarDecl *> values) override {
+    bool result = originalConsumer->consumePossiblyNotInScope(values);
+    return result;
+  }
+
+  /// Called right before looking at the parent scope of a BraceStmt.
+  ///
+  /// \return true if the lookup should be stopped at this point.
+  bool finishLookupInBraceStmt(BraceStmt *stmt) override {
+    return originalConsumer->finishLookupInBraceStmt(stmt);
+  }
+
+#ifndef NDEBUG
+  void startingNextLookupStep() override {
+    originalConsumer->startingNextLookupStep();
+  }
+  void finishingLookup(std::string input) const override {
+    originalConsumer->finishingLookup(input);
+  }
+  bool isTargetLookup() const override {
+    return originalConsumer->isTargetLookup();
+  }
+#endif
+};
+
 void ASTScope::unqualifiedLookup(
     SourceFile *SF, SourceLoc loc,
     namelookup::AbstractASTScopeDeclConsumer &consumer) {
@@ -48,7 +147,30 @@ void ASTScope::unqualifiedLookup(
 
   if (auto *s = SF->getASTContext().Stats)
     ++s->getFrontendCounters().NumASTScopeLookups;
-  ASTScopeImpl::unqualifiedLookup(SF, loc, consumer);
+
+  // Perform validation of SwiftLexicalLookup if option
+  // Feature::UnqualifiedLookupValidation is enabled and lookup was not
+  // performed in a macro.
+  if (SF->getASTContext().LangOpts.hasFeature(
+          Feature::UnqualifiedLookupValidation) &&
+      !SF->getEnclosingSourceFile()) {
+    LoggingASTScopeDeclConsumer loggingASTScopeDeclConsumer =
+        LoggingASTScopeDeclConsumer(&consumer);
+
+    ASTScopeImpl::unqualifiedLookup(SF, loc, loggingASTScopeDeclConsumer);
+
+    bool passed = swift_ASTGen_validateUnqualifiedLookup(
+        SF->getExportedSourceFile(), SF->getASTContext(), loc,
+        loggingASTScopeDeclConsumer.finishLookupInBraceStmt(nullptr),
+        BridgedArrayRef(loggingASTScopeDeclConsumer.recordedElements.data(),
+                        loggingASTScopeDeclConsumer.recordedElements.size()));
+
+    if (!passed) {
+      SF->getASTContext().Diags.diagnose(loc, diag::lookup_outputs_dont_match);
+    }
+  } else {
+    ASTScopeImpl::unqualifiedLookup(SF, loc, consumer);
+  }
 }
 
 llvm::SmallVector<LabeledStmt *, 4> ASTScope::lookupLabeledStmts(

lib/AST/CMakeLists.txt

@@ -171,4 +171,9 @@ target_link_libraries(swiftAST
   PUBLIC swiftBasic
   PRIVATE swiftMarkup)
 
+if (SWIFT_BUILD_SWIFT_SYNTAX)
+  target_link_libraries(swiftAST
+    PRIVATE swiftASTGen)
+endif()
+
 set_swift_llvm_is_available(swiftAST)

lib/AST/FeatureSet.cpp

@@ -102,6 +102,7 @@ UNINTERESTING_FEATURE(OpaqueTypeErasure)
 UNINTERESTING_FEATURE(PackageCMO)
 UNINTERESTING_FEATURE(ParserRoundTrip)
 UNINTERESTING_FEATURE(ParserValidation)
+UNINTERESTING_FEATURE(UnqualifiedLookupValidation)
 UNINTERESTING_FEATURE(ImplicitSome)
 UNINTERESTING_FEATURE(ParserASTGen)
 UNINTERESTING_FEATURE(BuiltinMacros)

lib/ASTGen/Package.swift

@@ -56,6 +56,7 @@ let package = Package(
       dependencies: [
         .product(name: "SwiftDiagnostics", package: "swift-syntax"),
         .product(name: "SwiftIfConfig", package: "swift-syntax"),
+        .product(name: "SwiftLexicalLookup", package: "swift-syntax"),
         .product(name: "SwiftOperators", package: "swift-syntax"),
         .product(name: "SwiftParser", package: "swift-syntax"),
         .product(name: "SwiftParserDiagnostics", package: "swift-syntax"),

lib/ASTGen/Sources/ASTGen/CMakeLists.txt

@@ -10,6 +10,7 @@ add_pure_swift_host_library(swiftASTGen STATIC CXX_INTEROP
   Exprs.swift
   Fingerprint.swift
   Generics.swift
+  LexicalLookup.swift
   Literals.swift
   ParameterClause.swift
   Patterns.swift
@@ -26,6 +27,7 @@ add_pure_swift_host_library(swiftASTGen STATIC CXX_INTEROP
     _CompilerRegexParser
     _CompilerSwiftSyntax
     _CompilerSwiftIfConfig
+    _CompilerSwiftLexicalLookup
     _CompilerSwiftOperators
     _CompilerSwiftSyntaxBuilder
     _CompilerSwiftParser