Cherry pick autocomplete fixes to tensorflow (#29707)

Cherry-picks some PRs necessary for https://bugs.swift.org/browse/SR-12076 into `tensorflow`.

This PR is only cherry-picks and llvm-project commit hash update, no additional changes.

Full explanation of the fix is in the PR description of the corresponding llvm-project PR: swiftlang/llvm-project#751.

Author: marcrasi

include/swift/AST/DiagnosticsParse.def

@@ -470,6 +470,8 @@ ERROR(expected_precedencegroup_relation,none,
       (StringRef))
 
 // SIL
+ERROR(expected_sil_keyword,none,
+      "expected SIL keyword", ())
 ERROR(inout_not_attribute, none,
       "@inout is no longer an attribute", ())
 ERROR(only_allowed_in_sil,none,

include/swift/AST/TypeCheckRequests.h

@@ -1959,17 +1959,15 @@ class DynamicallyReplacedDeclRequest
 
 class TypeCheckSourceFileRequest :
     public SimpleRequest<TypeCheckSourceFileRequest,
-                         bool (SourceFile *, unsigned),
-                         CacheKind::SeparatelyCached> {
+                         bool (SourceFile *), CacheKind::SeparatelyCached> {
 public:
   using SimpleRequest::SimpleRequest;
 
 private:
   friend SimpleRequest;
 
   // Evaluation.
-  llvm::Expected<bool> evaluate(Evaluator &evaluator,
-                                SourceFile *SF, unsigned StartElem) const;
+  llvm::Expected<bool> evaluate(Evaluator &evaluator, SourceFile *SF) const;
 
 public:
   // Separate caching.

include/swift/AST/TypeCheckerTypeIDZone.def

@@ -211,7 +211,7 @@ SWIFT_REQUEST(TypeChecker, HasDefaultInitRequest,
 SWIFT_REQUEST(TypeChecker, SynthesizeDefaultInitRequest,
               ConstructorDecl *(NominalTypeDecl *), Cached, NoLocationInfo)
 SWIFT_REQUEST(TypeChecker, TypeCheckSourceFileRequest,
-              bool(SouceFile *, unsigned), SeparatelyCached, NoLocationInfo)
+              bool(SouceFile *), SeparatelyCached, NoLocationInfo)
 SWIFT_REQUEST(TypeChecker, TypeWitnessRequest,
               TypeWitnessAndDecl(NormalProtocolConformance *,
                                  AssociatedTypeDecl *),

include/swift/Frontend/Frontend.h

@@ -653,8 +653,6 @@ class CompilerInstance {
   };
 
 private:
-  void createREPLFile(const ImplicitImports &implicitImports);
-
   void addMainFileToModule(const ImplicitImports &implicitImports);
 
   void performSemaUpTo(SourceFile::ASTStage_t LimitStage);

include/swift/Parse/Parser.h

@@ -413,7 +413,8 @@ class Parser {
          bool DelayBodyParsing = true);
   ~Parser();
 
-  bool isInSILMode() const { return SIL != nullptr; }
+  /// Returns true if the buffer being parsed is allowed to contain SIL.
+  bool isInSILMode() const;
 
   /// Calling this function to finalize libSyntax tree creation without destroying
   /// the parser instance.
@@ -430,9 +431,8 @@ class Parser {
                           PreviousLoc);
   }
 
-  ParserPosition getParserPosition(const PersistentParserState::ParserPos &Pos){
-    return ParserPosition(L->getStateForBeginningOfTokenLoc(Pos.Loc),
-                          Pos.PrevLoc);
+  ParserPosition getParserPosition(SourceLoc loc, SourceLoc previousLoc) {
+    return ParserPosition(L->getStateForBeginningOfTokenLoc(loc), previousLoc);
   }
 
   void restoreParserPosition(ParserPosition PP, bool enableDiagnostics = false) {
@@ -661,6 +661,9 @@ class Parser {
   /// Returns \c true if the parser hit the eof before finding matched '}'.
   bool skipBracedBlock();
 
+  /// Skip over SIL decls until we encounter the start of a Swift decl or eof.
+  void skipSILUntilSwiftDecl();
+
   /// If the parser is generating only a syntax tree, try loading the current
   /// node from a previously generated syntax tree.
   /// Returns \c true if the node has been loaded and inserted into the current
@@ -880,11 +883,18 @@ class Parser {
   //===--------------------------------------------------------------------===//
   // Decl Parsing
 
-  /// Return true if parser is at the start of a decl or decl-import.
-  bool isStartOfDecl();
+  /// Returns true if parser is at the start of a Swift decl or decl-import.
+  bool isStartOfSwiftDecl();
+
+  /// Returns true if the parser is at the start of a SIL decl.
+  bool isStartOfSILDecl();
 
+  /// Parse the top-level Swift decls into the source file.
   void parseTopLevel();
 
+  /// Parse the top-level SIL decls into the SIL module.
+  void parseTopLevelSIL();
+
   /// Flags that control the parsing of declarations.
   enum ParseDeclFlags {
     PD_Default              = 0,

include/swift/Parse/PersistentParserState.h

@@ -59,23 +59,12 @@ class CodeCompletionDelayedDeclState {
 /// Parser state persistent across multiple parses.
 class PersistentParserState {
 public:
-  struct ParserPos {
-    SourceLoc Loc;
-    SourceLoc PrevLoc;
-
-    bool isValid() const { return Loc.isValid(); }
-  };
-
-  bool InPoundLineEnvironment = false;
   // FIXME: When condition evaluation moves to a later phase, remove this bit
   // and adjust the client call 'performParseOnly'.
   bool PerformConditionEvaluation = true;
 private:
   swift::ScopeInfo ScopeInfo;
 
-  /// Parser sets this if it stopped parsing before the buffer ended.
-  ParserPosition MarkedPos;
-
   std::unique_ptr<CodeCompletionDelayedDeclState> CodeCompletionDelayedDeclStat;
 
   /// The local context for all top-level code.
@@ -113,19 +102,6 @@ class PersistentParserState {
   TopLevelContext &getTopLevelContext() {
     return TopLevelCode;
   }
-
-  void markParserPosition(ParserPosition Pos,
-                          bool InPoundLineEnvironment) {
-    MarkedPos = Pos;
-    this->InPoundLineEnvironment = InPoundLineEnvironment;
-  }
-
-  /// Returns the marked parser position and resets it.
-  ParserPosition takeParserPosition() {
-    ParserPosition Pos = MarkedPos;
-    MarkedPos = ParserPosition();
-    return Pos;
-  }
 };
 
 } // end namespace swift

include/swift/Subsystems.h

@@ -106,29 +106,21 @@ namespace swift {
 
   /// Parse a single buffer into the given source file.
   ///
-  /// If the source file is the main file, stop parsing after the next
-  /// stmt-brace-item with side-effects.
+  /// \param SF The file within the module being parsed.
   ///
-  /// \param SF the file within the module being parsed.
+  /// \param BufferID The buffer to parse from.
   ///
-  /// \param BufferID the buffer to parse from.
+  /// \param PersistentState If non-null the same PersistentState object can be
+  /// used to save parser state for code completion.
   ///
-  /// \param[out] Done set to \c true if end of the buffer was reached.
-  ///
-  /// \param SIL if non-null, we're parsing a SIL file.
-  ///
-  /// \param PersistentState if non-null the same PersistentState object can
-  /// be used to resume parsing or parse delayed function bodies.
-  void parseIntoSourceFile(SourceFile &SF, unsigned BufferID, bool *Done,
-                           SILParserState *SIL = nullptr,
+  /// \param DelayBodyParsing Whether parsing of type and function bodies can be
+  /// delayed.
+  void parseIntoSourceFile(SourceFile &SF, unsigned BufferID,
                            PersistentParserState *PersistentState = nullptr,
                            bool DelayBodyParsing = true);
 
-  /// Parse a single buffer into the given source file, until the full source
-  /// contents are parsed.
-  void parseIntoSourceFileFull(SourceFile &SF, unsigned BufferID,
-                               PersistentParserState *PersistentState = nullptr,
-                               bool DelayBodyParsing = true);
+  /// Parse a source file's SIL declarations into a given SIL module.
+  void parseSourceFileSIL(SourceFile &SF, SILParserState *sil);
 
   /// Finish the code completion.
   void performCodeCompletionSecondPass(PersistentParserState &PersistentState,
@@ -145,10 +137,7 @@ namespace swift {
 
   /// Once parsing is complete, this walks the AST to resolve imports, record
   /// operators, and do other top-level validation.
-  ///
-  /// \param StartElem Where to start for incremental name binding in the main
-  ///                  source file.
-  void performNameBinding(SourceFile &SF, unsigned StartElem = 0);
+  void performNameBinding(SourceFile &SF);
 
   /// Once type-checking is complete, this instruments code with calls to an
   /// intrinsic that record the expected values of local variables so they can
@@ -179,10 +168,7 @@ namespace swift {
 
   /// Once parsing and name-binding are complete, this walks the AST to resolve
   /// types and diagnose problems therein.
-  ///
-  /// \param StartElem Where to start for incremental type-checking in the main
-  /// source file.
-  void performTypeChecking(SourceFile &SF, unsigned StartElem = 0);
+  void performTypeChecking(SourceFile &SF);
 
   /// Now that we have type-checked an entire module, perform any type
   /// checking that requires the full module, e.g., Objective-C method

lib/Frontend/Frontend.cpp

@@ -727,7 +727,15 @@ void CompilerInstance::performSemaUpTo(SourceFile::ASTStage_t LimitStage) {
   const ImplicitImports implicitImports(*this);
 
   if (Invocation.getInputKind() == InputFileKind::SwiftREPL) {
-    createREPLFile(implicitImports);
+    // Create the initial empty REPL file. This only exists to feed in the
+    // implicit imports such as the standard library.
+    auto *replFile = createSourceFileForMainModule(
+        SourceFileKind::REPL, implicitImports.kind, /*BufferID*/ None);
+    addAdditionalInitialImportsTo(replFile, implicitImports);
+
+    // Given this file is empty, we can go ahead and just mark it as having been
+    // type checked.
+    replFile->ASTStage = SourceFile::TypeChecked;
     return;
   }
 
@@ -824,12 +832,6 @@ void CompilerInstance::getImplicitlyImportedModules(
   }
 }
 
-void CompilerInstance::createREPLFile(const ImplicitImports &implicitImports) {
-  auto *SingleInputFile = createSourceFileForMainModule(
-      Invocation.getSourceFileKind(), implicitImports.kind, None);
-  addAdditionalInitialImportsTo(SingleInputFile, implicitImports);
-}
-
 void CompilerInstance::addMainFileToModule(
     const ImplicitImports &implicitImports) {
   auto *MainFile = createSourceFileForMainModule(
@@ -913,14 +915,8 @@ void CompilerInstance::parseLibraryFile(
   auto DidSuppressWarnings = Diags.getSuppressWarnings();
   Diags.setSuppressWarnings(DidSuppressWarnings || !IsPrimary);
 
-  bool Done;
-  do {
-    // Parser may stop at some erroneous constructions like #else, #endif
-    // or '}' in some cases, continue parsing until we are done
-    parseIntoSourceFile(*NextInput, BufferID, &Done, nullptr,
-                        PersistentState.get(),
-                        /*DelayedBodyParsing=*/!IsPrimary);
-  } while (!Done);
+  parseIntoSourceFile(*NextInput, BufferID, PersistentState.get(),
+                      /*DelayedBodyParsing=*/!IsPrimary);
 
   Diags.setSuppressWarnings(DidSuppressWarnings);
 
@@ -953,6 +949,7 @@ bool CompilerInstance::parsePartialModulesAndLibraryFiles(
 
 void CompilerInstance::parseAndTypeCheckMainFileUpTo(
     SourceFile::ASTStage_t LimitStage) {
+  assert(LimitStage >= SourceFile::NameBound);
   FrontendStatsTracer tracer(Context->Stats,
                              "parse-and-typecheck-main-file");
   bool mainIsPrimary =
@@ -965,45 +962,23 @@ void CompilerInstance::parseAndTypeCheckMainFileUpTo(
   auto DidSuppressWarnings = Diags.getSuppressWarnings();
   Diags.setSuppressWarnings(DidSuppressWarnings || !mainIsPrimary);
 
-  SILParserState SILContext(TheSILModule.get());
-  unsigned CurTUElem = 0;
-  bool Done;
-  do {
-    // Pump the parser multiple times if necessary.  It will return early
-    // after parsing any top level code in a main module, or in SIL mode when
-    // there are chunks of swift decls (e.g. imports and types) interspersed
-    // with 'sil' definitions.
-    parseIntoSourceFile(MainFile, MainFile.getBufferID().getValue(), &Done,
-                        TheSILModule ? &SILContext : nullptr,
-                        PersistentState.get(),
-                        !mainIsPrimary);
-
-    // For SIL we actually have to interleave parsing and type checking
-    // because the SIL parser expects to see fully type checked declarations.
-    if (TheSILModule) {
-      if (Done || CurTUElem < MainFile.getTopLevelDecls().size()) {
-        assert(mainIsPrimary);
-        performTypeChecking(MainFile, CurTUElem);
-      }
-    }
+  // Parse the Swift decls into the source file.
+  parseIntoSourceFile(MainFile, MainBufferID, PersistentState.get(),
+                      /*delayBodyParsing*/ !mainIsPrimary);
 
-    CurTUElem = MainFile.getTopLevelDecls().size();
-  } while (!Done);
-
-  if (!TheSILModule) {
-    if (mainIsPrimary) {
-      switch (LimitStage) {
-      case SourceFile::Parsing:
-      case SourceFile::Parsed:
-        llvm_unreachable("invalid limit stage");
-      case SourceFile::NameBound:
-        performNameBinding(MainFile);
-        break;
-      case SourceFile::TypeChecked:
-        performTypeChecking(MainFile);
-        break;
-      }
-    }
+  // For a primary, also perform type checking if needed. Otherwise, just do
+  // name binding.
+  if (mainIsPrimary && LimitStage >= SourceFile::TypeChecked) {
+    performTypeChecking(MainFile);
+  } else {
+    assert(!TheSILModule && "Should perform type checking for SIL");
+    performNameBinding(MainFile);
+  }
+
+  // Parse the SIL decls if needed.
+  if (TheSILModule) {
+    SILParserState SILContext(TheSILModule.get());
+    parseSourceFileSIL(MainFile, &SILContext);
   }
 
   Diags.setSuppressWarnings(DidSuppressWarnings);
@@ -1012,12 +987,6 @@ void CompilerInstance::parseAndTypeCheckMainFileUpTo(
       Invocation.getFrontendOptions().DebuggerTestingTransform) {
     performDebuggerTestingTransform(MainFile);
   }
-
-  if (!TheSILModule) {
-    if (!mainIsPrimary) {
-      performNameBinding(MainFile);
-    }
-  }
 }
 
 static void
@@ -1109,8 +1078,8 @@ void CompilerInstance::performParseOnly(bool EvaluateConditionals,
         SourceFileKind::Library, SourceFile::ImplicitModuleImportKind::None,
         BufferID);
 
-    parseIntoSourceFileFull(*NextInput, BufferID, PersistentState.get(),
-                            shouldDelayBodies(BufferID));
+    parseIntoSourceFile(*NextInput, BufferID, PersistentState.get(),
+                        shouldDelayBodies(BufferID));
   }
 
   // Now parse the main file.
@@ -1120,8 +1089,8 @@ void CompilerInstance::performParseOnly(bool EvaluateConditionals,
     MainFile.SyntaxParsingCache = Invocation.getMainFileSyntaxParsingCache();
     assert(MainBufferID == MainFile.getBufferID());
 
-    parseIntoSourceFileFull(MainFile, MainBufferID, PersistentState.get(),
-                            shouldDelayBodies(MainBufferID));
+    parseIntoSourceFile(MainFile, MainBufferID, PersistentState.get(),
+                        shouldDelayBodies(MainBufferID));
   }
 
   assert(Context->LoadedModules.size() == 1 &&

lib/IDE/CompletionInstance.cpp

@@ -217,7 +217,7 @@ bool CompletionInstance::performCachedOperaitonIfPossible(
   newSF->enableInterfaceHash();
   // Ensure all non-function-body tokens are hashed into the interface hash
   Ctx->LangOpts.EnableTypeFingerprints = false;
-  parseIntoSourceFileFull(*newSF, tmpBufferID, &newState);
+  parseIntoSourceFile(*newSF, tmpBufferID, &newState);
   // Couldn't find any completion token?
   if (!newState.hasCodeCompletionDelayedDeclState())
     return false;