Rework the integrated REPL to use separate modules for every line

...like LLDB does, instead of parsing into a single SourceFile.
This does break some functionality:

- no more :dump_ast
- no redeclaration checking, but no shadowing either---redeclarations
  just become ambiguous
- pretty much requires EnableAccessControl to be off, since we don't
  walk decls to promote them to 'public'

...but it allows us to remove a bit of longstanding support for
type-checking / SILGen-ing / IRGen-ing only part of a SourceFile that
was only used by the integrated REPL.

...which, need I remind everyone, is still /deprecated/...but sometimes
convenient. So most of it still works.

Author: jrose-apple

lib/AST/Module.cpp

@@ -1305,6 +1305,12 @@ SourceFile::collectLinkLibraries(ModuleDecl::LinkLibraryCallback callback) const
     if (next->getName() == getParentModule()->getName())
       return true;
 
+    // Hack: Assume other REPL files already have their libraries linked.
+    if (!next->getFiles().empty())
+      if (auto *nextSource = dyn_cast<SourceFile>(next->getFiles().front()))
+        if (nextSource->Kind == SourceFileKind::REPL)
+          return true;
+
     next->collectLinkLibraries(callback);
     return true;
   });

lib/Immediate/REPL.cpp

@@ -46,20 +46,6 @@ using namespace swift::immediate;
 
 namespace {
 
-class REPLContext {
-public:
-  /// The SourceMgr buffer ID of the REPL input.
-  unsigned CurBufferID;
-  
-  /// The index into the source file's Decls at which to start
-  /// typechecking the next REPL input.
-  unsigned CurElem;
-
-  /// The index into the source file's Decls at which to start
-  /// irgenning the next REPL input.
-  unsigned CurIRGenElem;
-};
-
 enum class REPLInputKind : int {
   /// The REPL got a "quit" signal.
   REPLQuit,
@@ -168,27 +154,49 @@ static void convertToUTF8(llvm::ArrayRef<wchar_t> wide,
 
 #if HAVE_UNICODE_LIBEDIT
 
-static bool appendToREPLFile(SourceFile &SF,
-                             PersistentParserState &PersistentState,
-                             REPLContext &RC,
-                             std::unique_ptr<llvm::MemoryBuffer> Buffer) {
-  assert(SF.Kind == SourceFileKind::REPL && "Can't append to a non-REPL file");
-
-  SourceManager &SrcMgr = SF.getParentModule()->getASTContext().SourceMgr;
-  RC.CurBufferID = SrcMgr.addNewSourceBuffer(std::move(Buffer));
+static ModuleDecl *
+typeCheckREPLInput(ModuleDecl *MostRecentModule, StringRef Name,
+                   PersistentParserState &PersistentState,
+                   std::unique_ptr<llvm::MemoryBuffer> Buffer) {
+  using ImplicitModuleImportKind = SourceFile::ImplicitModuleImportKind;
+  assert(MostRecentModule);
+  ASTContext &Ctx = MostRecentModule->getASTContext();
+
+  auto REPLModule = ModuleDecl::create(Ctx.getIdentifier(Name), Ctx);
+  auto BufferID = Ctx.SourceMgr.addNewSourceBuffer(std::move(Buffer));
+  auto ImportKind = ImplicitModuleImportKind::None;
+  auto &REPLInputFile = *new (Ctx) SourceFile(*REPLModule, SourceFileKind::REPL,
+                                              BufferID, ImportKind);
+  REPLModule->addFile(REPLInputFile);
+
+  ModuleDecl::ImportedModule ImportOfMostRecentModule{
+      /*AccessPath*/{}, MostRecentModule};
+  REPLInputFile.addImports(std::make_pair(ImportOfMostRecentModule,
+                                          SourceFile::ImportOptions()));
+
+  SmallVector<ModuleDecl::ImportedModule, 8> Imports;
+  MostRecentModule->getImportedModules(Imports,
+                                       ModuleDecl::ImportFilter::Private);
+  if (!Imports.empty()) {
+    SmallVector<std::pair<ModuleDecl::ImportedModule,
+                          SourceFile::ImportOptions>, 8> ImportsWithOptions;
+    for (auto Import : Imports) {
+      ImportsWithOptions.emplace_back(Import,
+                                      SourceFile::ImportFlags::Exported);
+    }
+    REPLInputFile.addImports(ImportsWithOptions);
+  }
 
   bool FoundAnySideEffects = false;
-  unsigned CurElem = RC.CurElem;
   bool Done;
   do {
     FoundAnySideEffects |=
-        parseIntoSourceFile(SF, RC.CurBufferID, &Done, nullptr,
+        parseIntoSourceFile(REPLInputFile, BufferID, &Done, nullptr,
                             &PersistentState);
-    performTypeChecking(SF, PersistentState.getTopLevelContext(), None,
-                        CurElem);
-    CurElem = SF.Decls.size();
   } while (!Done);
-  return FoundAnySideEffects;
+  performTypeChecking(REPLInputFile, PersistentState.getTopLevelContext(),
+                      /*Options*/None);
+  return REPLModule;
 }
 
 /// An arbitrary, otherwise-unused char value that editline interprets as
@@ -327,8 +335,6 @@ class REPLInput {
     fflush(stdout);
     el_end(e);
   }
-  
-  SourceFile &getREPLInputFile();
 
   REPLInputKind getREPLInput(SmallVectorImpl<char> &Result) {
     ide::SourceCompleteResult SCR;
@@ -667,6 +673,8 @@ class REPLInput {
     if (trimmed > 0)
       llvm::outs() << "  (and " << trimmed << " more)\n";
   }
+
+  SourceFile &getFileForCodeCompletion();
 
   unsigned char onComplete(int ch) {
     const LineInfoW *line = el_wline(e);
@@ -677,7 +685,7 @@ class REPLInput {
 
     if (!completions) {
       // If we aren't currently working with a completion set, generate one.
-      completions.populate(getREPLInputFile(), Prefix);
+      completions.populate(getFileForCodeCompletion(), Prefix);
       // Display the common root of the found completions and beep unless we
       // found a unique one.
       insertStringRef(completions.getRoot());
@@ -736,11 +744,7 @@ static void printOrDumpDecl(Decl *d, PrintOrDump which) {
 /// The compiler and execution environment for the REPL.
 class REPLEnvironment {
   CompilerInstance &CI;
-
-public:
-  SourceFile &REPLInputFile;
-
-private:
+  ModuleDecl *MostRecentModule;
   ProcessCmdLine CmdLine;
   llvm::SmallPtrSet<swift::ModuleDecl *, 8> ImportedModules;
   SmallVector<llvm::Function*, 8> InitFns;
@@ -757,8 +761,8 @@ class REPLEnvironment {
   const SILOptions SILOpts;
 
   REPLInput Input;
-  REPLContext RC;
   PersistentParserState PersistentState;
+  unsigned NextLineNumber = 0;
 
 private:
 
@@ -858,17 +862,18 @@ class REPLEnvironment {
 
   bool executeSwiftSource(llvm::StringRef Line, const ProcessCmdLine &CmdLine) {
     // Parse the current line(s).
-    auto InputBuf = std::unique_ptr<llvm::MemoryBuffer>(
-        llvm::MemoryBuffer::getMemBufferCopy(Line, "<REPL Input>"));
-    bool ShouldRun = appendToREPLFile(REPLInputFile, PersistentState, RC,
-                                      std::move(InputBuf));
+    auto InputBuf = llvm::MemoryBuffer::getMemBufferCopy(Line, "<REPL Input>");
+    SmallString<8> Name{"REPL_"};
+    llvm::raw_svector_ostream(Name) << NextLineNumber;
+    ++NextLineNumber;
+    ModuleDecl *M = typeCheckREPLInput(MostRecentModule, Name, PersistentState,
+                                       std::move(InputBuf));
 
     // SILGen the module and produce SIL diagnostics.
     std::unique_ptr<SILModule> sil;
 
     if (!CI.getASTContext().hadError()) {
-      sil = performSILGeneration(REPLInputFile, CI.getSILOptions(),
-                                 RC.CurIRGenElem);
+      sil = performSILGeneration(M, CI.getSILOptions());
       runSILDiagnosticPasses(*sil);
       runSILLoweringPasses(*sil);
     }
@@ -878,34 +883,23 @@ class REPLEnvironment {
         return false;
 
       CI.getASTContext().Diags.resetHadAnyError();
-      while (REPLInputFile.Decls.size() > RC.CurElem)
-        REPLInputFile.Decls.pop_back();
-      
+
       // FIXME: Handling of "import" declarations?  Is there any other
       // state which needs to be reset?
 
       return true;
     }
-    
-    RC.CurElem = REPLInputFile.Decls.size();
-    
+    MostRecentModule = M;
+
     DumpSource += Line;
-    
-    // If we didn't see an expression, statement, or decl which might have
-    // side-effects, keep reading.
-    if (!ShouldRun)
-      return true;
 
-    const PrimarySpecificPaths PSPs =
-        CI.getPrimarySpecificPathsForAtMostOnePrimary();
     // IRGen the current line(s).
     // FIXME: We shouldn't need to use the global context here, but
     // something is persisting across calls to performIRGeneration.
     auto LineModule = performIRGeneration(
-        IRGenOpts, REPLInputFile, std::move(sil), "REPLLine", PSPs,
-        getGlobalLLVMContext(), RC.CurIRGenElem);
-    RC.CurIRGenElem = RC.CurElem;
-    
+        IRGenOpts, M, std::move(sil), "REPLLine", PrimarySpecificPaths(),
+        getGlobalLLVMContext(), /*parallelOutputFilenames*/{});
+
     if (CI.getASTContext().hadError())
       return false;
 
@@ -929,7 +923,7 @@ class REPLEnvironment {
     llvm::Function *DumpModuleMain = DumpModule.getFunction("main");
     DumpModuleMain->setName("repl.line");
 
-    if (autolinkImportedModules(getMainModule(), IRGenOpts))
+    if (autolinkImportedModules(M, IRGenOpts))
       return false;
 
     llvm::Module *TempModule = NewModule.get();
@@ -959,8 +953,7 @@ class REPLEnvironment {
                   llvm::LLVMContext &LLVMCtx,
                   bool ParseStdlib)
     : CI(CI),
-      REPLInputFile(CI.getMainModule()->
-                      getMainSourceFile(SourceFileKind::REPL)),
+      MostRecentModule(CI.getMainModule()),
       CmdLine(CmdLine),
       RanGlobalInitializers(false),
       LLVMContext(LLVMCtx),
@@ -969,20 +962,19 @@ class REPLEnvironment {
       IRGenOpts(),
       SILOpts(),
       Input(*this),
-      RC{
-        /*BufferID*/ 0U,
-        /*CurElem*/ 0,
-        /*CurIRGenElem*/ 0
-      },
       PersistentState(CI.getASTContext())
   {
     ASTContext &Ctx = CI.getASTContext();
-    if (!loadSwiftRuntime(Ctx.SearchPathOpts.RuntimeLibraryPath)) {
-      CI.getDiags().diagnose(SourceLoc(),
-                             diag::error_immediate_mode_missing_stdlib);
-      return;
+    Ctx.LangOpts.EnableAccessControl = false;
+    if (!ParseStdlib) {
+      if (!loadSwiftRuntime(Ctx.SearchPathOpts.RuntimeLibraryPath)) {
+        CI.getDiags().diagnose(SourceLoc(),
+                               diag::error_immediate_mode_missing_stdlib);
+        return;
+      }
+      tryLoadLibraries(CI.getLinkLibraries(), Ctx.SearchPathOpts,
+                       CI.getDiags());
     }
-    tryLoadLibraries(CI.getLinkLibraries(), Ctx.SearchPathOpts, CI.getDiags());
 
     llvm::EngineBuilder builder{std::unique_ptr<llvm::Module>{Module}};
     std::string ErrorMsg;
@@ -1008,6 +1000,10 @@ class REPLEnvironment {
     IRGenOpts.DebugInfoLevel = IRGenDebugInfoLevel::None;
     IRGenOpts.DebugInfoFormat = IRGenDebugInfoFormat::None;
 
+    // The very first module is a dummy.
+    CI.getMainModule()->getMainSourceFile(SourceFileKind::REPL).ASTStage =
+        SourceFile::TypeChecked;
+
     if (!ParseStdlib) {
       // Force standard library to be loaded immediately.  This forces any
       // errors to appear upfront, and helps eliminate some nasty lag after the
@@ -1017,14 +1013,13 @@ class REPLEnvironment {
       auto Buffer =
           llvm::MemoryBuffer::getMemBufferCopy(WarmUpStmt,
                                                "<REPL Initialization>");
-      appendToREPLFile(REPLInputFile, PersistentState, RC, std::move(Buffer));
+      (void)typeCheckREPLInput(MostRecentModule, "__Warmup", PersistentState,
+                               std::move(Buffer));
 
       if (Ctx.hadError())
         return;
     }
-    
-    RC.CurElem = RC.CurIRGenElem = REPLInputFile.Decls.size();
-    
+
     if (llvm::sys::Process::StandardInIsUserInput())
       llvm::outs() <<
           "***  You are running Swift's integrated REPL,  ***\n"
@@ -1034,13 +1029,14 @@ class REPLEnvironment {
           "***  Type ':help' for assistance.              ***\n";
   }
 
-  swift::ModuleDecl *getMainModule() const {
-    return REPLInputFile.getParentModule();
-  }
   StringRef getDumpSource() const { return DumpSource; }
 
   /// Get the REPLInput object owned by the REPL instance.
   REPLInput &getInput() { return Input; }
+
+  SourceFile &getFileForCodeCompletion() {
+    return MostRecentModule->getMainSourceFile(SourceFileKind::REPL);
+  }
 
   /// Responds to a REPL input. Returns true if the repl should continue,
   /// false if it should quit.
@@ -1070,8 +1066,6 @@ class REPLEnvironment {
                "  :constraints debug (on|off) - turn on/off the debug "
                    "output for the constraint-based type checker\n"
                "  :dump_ir - dump the LLVM IR generated by the REPL\n"
-               "  :dump_ast - dump the AST representation of"
-                   " the REPL input\n"
                "  :dump_decl <name> - dump the AST representation of the "
                    "named declarations\n"
                "  :dump_source - dump the user input (ignoring"
@@ -1086,17 +1080,17 @@ class REPLEnvironment {
           return false;
         } else if (L.peekNextToken().getText() == "dump_ir") {
           DumpModule.print(llvm::dbgs(), nullptr, false, true);
-        } else if (L.peekNextToken().getText() == "dump_ast") {
-          REPLInputFile.dump();
         } else if (L.peekNextToken().getText() == "dump_decl" ||
                    L.peekNextToken().getText() == "print_decl") {
           PrintOrDump doPrint = (L.peekNextToken().getText() == "print_decl")
             ? PrintOrDump::Print : PrintOrDump::Dump;
           L.lex(Tok);
           L.lex(Tok);
           ASTContext &ctx = CI.getASTContext();
-          UnqualifiedLookup lookup(ctx.getIdentifier(Tok.getText()),
-                                   &REPLInputFile, nullptr);
+          SourceFile &SF =
+              MostRecentModule->getMainSourceFile(SourceFileKind::REPL);
+          UnqualifiedLookup lookup(ctx.getIdentifier(Tok.getText()), &SF,
+                                   nullptr);
           for (auto result : lookup.Results) {
             printOrDumpDecl(result.getValueDecl(), doPrint);
 
@@ -1201,7 +1195,9 @@ class REPLEnvironment {
   }
 };
 
-inline SourceFile &REPLInput::getREPLInputFile() { return Env.REPLInputFile; }
+inline SourceFile &REPLInput::getFileForCodeCompletion() {
+  return Env.getFileForCodeCompletion();
+}
 
 void PrettyStackTraceREPL::print(llvm::raw_ostream &out) const {
   out << "while processing REPL source:\n";

lib/Sema/TypeCheckREPL.cpp

@@ -308,7 +308,7 @@ void REPLChecker::processREPLTopLevelExpr(Expr *E) {
   E = TC.coerceToRValue(E);
 
   // Create the meta-variable, let the typechecker name it.
-  Identifier name = TC.getNextResponseVariableName(SF.getParentModule());
+  Identifier name = TC.getNextResponseVariableName(&SF);
   VarDecl *vd = new (Context) VarDecl(/*IsStatic*/false,
                                       VarDecl::Specifier::Let,
                                       /*IsCaptureList*/false, E->getStartLoc(),
@@ -456,5 +456,6 @@ void TypeChecker::processREPLTopLevel(SourceFile &SF, TopLevelContext &TLC,
   }
 
   contextualizeTopLevelCode(TLC, llvm::makeArrayRef(SF.Decls).slice(FirstDecl));
+  SF.clearLookupCache();
 }
 

test/Interpreter/SDK/Cocoa_repl.swift

@@ -23,7 +23,7 @@ extension CGRect: Q {
 // non-imported types.
 struct Empty {}
 let _: Optional = Empty()
-// CHECK: Optional(REPL.Empty())
+// CHECK: Optional(REPL_{{.+}}.Empty())
 let _: Optional = CGPoint.zero
 // CHECK: Optional((0.0, 0.0))
 let _: Optional = NSString.availableStringEncodings

test/Interpreter/repl.swift

@@ -157,17 +157,17 @@ var b = a.slice[3..<5]
 
 struct Inner<T> {}
 struct Outer<T> { var inner : Inner<T> }
-Outer<Int>(inner: Inner()) // CHECK: Outer<Int> = REPL.Outer
+Outer<Int>(inner: Inner()) // CHECK: Outer<Int> = REPL_{{.+}}.Outer
 
 struct ContainsSlice { var slice : [Int] }
-ContainsSlice(slice: [1, 2, 3]) // CHECK: ContainsSlice = REPL.ContainsSlice
+ContainsSlice(slice: [1, 2, 3]) // CHECK: ContainsSlice = REPL_{{.+}}.ContainsSlice
 
 struct ContainsGenericSlice<T> { var slice : [T] }
-ContainsGenericSlice(slice: [1, 2, 3]) // CHECK: ContainsGenericSlice<Int> = REPL.ContainsGenericSlice
-ContainsGenericSlice(slice: [(1, 2), (3, 4)]) // CHECK: ContainsGenericSlice<(Int, Int)> = REPL.ContainsGenericSlice
+ContainsGenericSlice(slice: [1, 2, 3]) // CHECK: ContainsGenericSlice<Int> = REPL_{{.+}}.ContainsGenericSlice
+ContainsGenericSlice(slice: [(1, 2), (3, 4)]) // CHECK: ContainsGenericSlice<(Int, Int)> = REPL_{{.+}}.ContainsGenericSlice
 
 struct ContainsContainsSlice { var containsSlice : ContainsSlice }
-ContainsContainsSlice(containsSlice: ContainsSlice(slice: [1, 2, 3])) // CHECK: ContainsContainsSlice = REPL.ContainsContainsSlice
+ContainsContainsSlice(containsSlice: ContainsSlice(slice: [1, 2, 3])) // CHECK: ContainsContainsSlice = REPL_{{.+}}.ContainsContainsSlice
 
 protocol Proto {
   func foo()