Code Size: Outline copy addr instruction

docs/ABI/Mangling.rst

@@ -83,6 +83,10 @@ Globals
   global ::= type 'We' // Outlined Consume Function Type
   global ::= type 'Wr' // Outlined Retain Function Type
   global ::= type 'Ws' // Outlined Release Function Type
+  global ::= type 'Wb' // Outlined InitializeWithTake Function Type
+  global ::= type 'Wc' // Outlined InitializeWithCopy Function Type
+  global ::= type 'Wd' // Outlined AssignWithTake Function Type
+  global ::= type 'Wf' // Outlined AssignWithCopy Function Type
 
   assoc_type_path ::= identifier '_' identifier*
 

include/swift/Demangling/DemangleNodes.def

@@ -189,6 +189,10 @@ NODE(OutlinedCopy)
 NODE(OutlinedConsume)
 NODE(OutlinedRetain)
 NODE(OutlinedRelease)
+NODE(OutlinedInitializeWithTake)
+NODE(OutlinedInitializeWithCopy)
+NODE(OutlinedAssignWithTake)
+NODE(OutlinedAssignWithCopy)
 NODE(OutlinedVariable)
 NODE(AssocTypePath)
 #undef CONTEXT_NODE

include/swift/Serialization/ModuleFormat.h

@@ -54,7 +54,7 @@ const uint16_t VERSION_MAJOR = 0;
 /// in source control, you should also update the comment to briefly
 /// describe what change you made. The content of this comment isn't important;
 /// it just ensures a conflict if two people change the module format.
-const uint16_t VERSION_MINOR = 375; // Last change: added enum witness
+const uint16_t VERSION_MINOR = 376; // Last change: Outlined CopyAddr
 
 using DeclID = PointerEmbeddedInt<unsigned, 31>;
 using DeclIDField = BCFixed<31>;

lib/Demangling/Demangler.cpp

@@ -1687,6 +1687,23 @@ NodePointer Demangler::demangleWitness() {
       return createWithChild(Node::Kind::OutlinedRelease,
                              popNode(Node::Kind::Type));
     }
+    case 'b': {
+      return createWithChild(Node::Kind::OutlinedInitializeWithTake,
+                             popNode(Node::Kind::Type));
+    }
+    case 'c': {
+      return createWithChild(Node::Kind::OutlinedInitializeWithCopy,
+                             popNode(Node::Kind::Type));
+    }
+    case 'd': {
+      return createWithChild(Node::Kind::OutlinedAssignWithTake,
+                             popNode(Node::Kind::Type));
+    }
+    case 'f': {
+      return createWithChild(Node::Kind::OutlinedAssignWithCopy,
+                             popNode(Node::Kind::Type));
+    }
+
     default:
       return nullptr;
   }

lib/Demangling/NodePrinter.cpp

@@ -416,6 +416,10 @@ class NodePrinter {
     case Node::Kind::OutlinedConsume:
     case Node::Kind::OutlinedRetain:
     case Node::Kind::OutlinedRelease:
+    case Node::Kind::OutlinedInitializeWithTake:
+    case Node::Kind::OutlinedInitializeWithCopy:
+    case Node::Kind::OutlinedAssignWithTake:
+    case Node::Kind::OutlinedAssignWithCopy:
     case Node::Kind::OutlinedVariable:
     case Node::Kind::AssocTypePath:
       return false;
@@ -809,6 +813,22 @@ NodePointer NodePrinter::print(NodePointer Node, bool asPrefixContext) {
     Printer << "outlined release of ";
     print(Node->getChild(0));
     return nullptr;
+  case Node::Kind::OutlinedInitializeWithTake:
+    Printer << "outlined init with take of ";
+    print(Node->getChild(0));
+    return nullptr;
+  case Node::Kind::OutlinedInitializeWithCopy:
+    Printer << "outlined init with copy of ";
+    print(Node->getChild(0));
+    return nullptr;
+  case Node::Kind::OutlinedAssignWithTake:
+    Printer << "outlined assign with take of ";
+    print(Node->getChild(0));
+    return nullptr;
+  case Node::Kind::OutlinedAssignWithCopy:
+    Printer << "outlined assign with copy of ";
+    print(Node->getChild(0));
+    return nullptr;
   case Node::Kind::OutlinedVariable:
     Printer << "outlined variable #" << Node->getIndex() << " of ";
     return nullptr;

lib/Demangling/OldRemangler.cpp

@@ -1700,6 +1700,26 @@ void Remangler::mangleOutlinedRelease(Node *node) {
   mangleSingleChildNode(node);
 }
 
+void Remangler::mangleOutlinedInitializeWithTake(Node *node) {
+  Out << "Wb";
+  mangleSingleChildNode(node);
+}
+
+void Remangler::mangleOutlinedInitializeWithCopy(Node *node) {
+  Out << "Wc";
+  mangleSingleChildNode(node);
+}
+
+void Remangler::mangleOutlinedAssignWithTake(Node *node) {
+  Out << "Wd";
+  mangleSingleChildNode(node);
+}
+
+void Remangler::mangleOutlinedAssignWithCopy(Node *node) {
+  Out << "Wf";
+  mangleSingleChildNode(node);
+}
+
 void Remangler::mangleOutlinedVariable(Node *node) {
   Out << "Tv" << node->getIndex();
   mangleSingleChildNode(node);

lib/Demangling/Remangler.cpp

@@ -1747,6 +1747,26 @@ void Remangler::mangleOutlinedRelease(Node *node) {
   Buffer << "Ws";
 }
 
+void Remangler::mangleOutlinedInitializeWithTake(Node *node) {
+  mangleSingleChildNode(node);
+  Buffer << "Wb";
+}
+
+void Remangler::mangleOutlinedInitializeWithCopy(Node *node) {
+  mangleSingleChildNode(node);
+  Buffer << "Wc";
+}
+
+void Remangler::mangleOutlinedAssignWithTake(Node *node) {
+  mangleSingleChildNode(node);
+  Buffer << "Wd";
+}
+
+void Remangler::mangleOutlinedAssignWithCopy(Node *node) {
+  mangleSingleChildNode(node);
+  Buffer << "Wf";
+}
+
 void Remangler::mangleOutlinedVariable(Node *node) {
   Buffer << "Tv";
   mangleIndex(node->getIndex());

lib/IRGen/GenDecl.cpp

@@ -19,6 +19,7 @@
 #include "swift/AST/Decl.h"
 #include "swift/AST/DiagnosticEngine.h"
 #include "swift/AST/DiagnosticsIRGen.h"
+#include "swift/AST/GenericEnvironment.h"
 #include "swift/AST/GenericSignature.h"
 #include "swift/AST/IRGenOptions.h"
 #include "swift/AST/Module.h"
@@ -39,10 +40,10 @@
 #include "clang/AST/GlobalDecl.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/IR/GlobalAlias.h"
+#include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/TypeBuilder.h"
 #include "llvm/IR/Value.h"
-#include "llvm/IR/InlineAsm.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/ConvertUTF.h"
 #include "llvm/Support/Path.h"
@@ -3517,6 +3518,7 @@ llvm::Constant *IRGenModule::getOrCreateRetainFunction(const TypeInfo &objectTI,
   return getOrCreateHelperFunction(
       funcName, llvmType, argTys,
       [&](IRGenFunction &IGF) {
+        IGF.setInOutlinedFunction();
         auto it = IGF.CurFn->arg_begin();
         Address addr(&*it++, loadableTI->getFixedAlignment());
         Explosion loaded;
@@ -3540,6 +3542,7 @@ IRGenModule::getOrCreateReleaseFunction(const TypeInfo &objectTI, Type t,
   return getOrCreateHelperFunction(
       funcName, llvmType, argTys,
       [&](IRGenFunction &IGF) {
+        IGF.setInOutlinedFunction();
         auto it = IGF.CurFn->arg_begin();
         Address addr(&*it++, loadableTI->getFixedAlignment());
         Explosion loaded;
@@ -3549,3 +3552,87 @@ IRGenModule::getOrCreateReleaseFunction(const TypeInfo &objectTI, Type t,
       },
       true /*setIsNoInline*/);
 }
+
+void IRGenModule::generateCallToOutlinedCopyAddr(
+    IRGenFunction &IGF, const TypeInfo &objectTI, Address dest, Address src,
+    SILType T, const OutlinedCopyAddrFunction MethodToCall) {
+  llvm::Type *llvmType = dest->getType();
+  auto *outlinedF = (this->*MethodToCall)(objectTI, llvmType, T);
+  llvm::Value *args[] = {src.getAddress(), dest.getAddress()};
+  llvm::CallInst *call = IGF.Builder.CreateCall(outlinedF, args);
+  call->setCallingConv(DefaultCC);
+}
+
+llvm::Constant *IRGenModule::getOrCreateOutlinedCopyAddrHelperFunction(
+    const TypeInfo &objectTI, llvm::Type *llvmType, SILType addrTy,
+    std::string funcName,
+    llvm::function_ref<void(const TypeInfo &objectTI, IRGenFunction &IGF,
+                            Address dest, Address src, SILType T)>
+        Generate) {
+  llvm::Type *argTys[] = {llvmType, llvmType};
+  return getOrCreateHelperFunction(
+      funcName, llvmType, argTys,
+      [&](IRGenFunction &IGF) {
+        IGF.setInOutlinedFunction();
+        auto it = IGF.CurFn->arg_begin();
+        Address src(&*it++, objectTI.getBestKnownAlignment());
+        Address dest(&*it++, objectTI.getBestKnownAlignment());
+        Generate(objectTI, IGF, dest, src, addrTy);
+        IGF.Builder.CreateRet(dest.getAddress());
+      },
+      true /*setIsNoInline*/);
+}
+
+llvm::Constant *IRGenModule::getOrCreateOutlinedInitializeWithTakeFunction(
+    const TypeInfo &objectTI, llvm::Type *llvmType, SILType addrTy) {
+  IRGenMangler mangler;
+  CanType canType = addrTy.getObjectType().getSwiftRValueType();
+  std::string funcName =
+      mangler.mangleOutlinedInitializeWithTakeFunction(canType);
+  auto GenFunc = [this](const TypeInfo &objectTI, IRGenFunction &IGF,
+                        Address dest, Address src, SILType T) {
+    objectTI.initializeWithTake(IGF, dest, src, T);
+  };
+  return getOrCreateOutlinedCopyAddrHelperFunction(objectTI, llvmType, addrTy,
+                                                   funcName, GenFunc);
+}
+
+llvm::Constant *IRGenModule::getOrCreateOutlinedInitializeWithCopyFunction(
+    const TypeInfo &objectTI, llvm::Type *llvmType, SILType addrTy) {
+  IRGenMangler mangler;
+  CanType canType = addrTy.getObjectType().getSwiftRValueType();
+  std::string funcName =
+      mangler.mangleOutlinedInitializeWithCopyFunction(canType);
+  auto GenFunc = [this](const TypeInfo &objectTI, IRGenFunction &IGF,
+                        Address dest, Address src, SILType T) {
+    objectTI.initializeWithCopy(IGF, dest, src, T);
+  };
+  return getOrCreateOutlinedCopyAddrHelperFunction(objectTI, llvmType, addrTy,
+                                                   funcName, GenFunc);
+}
+
+llvm::Constant *IRGenModule::getOrCreateOutlinedAssignWithTakeFunction(
+    const TypeInfo &objectTI, llvm::Type *llvmType, SILType addrTy) {
+  IRGenMangler mangler;
+  CanType canType = addrTy.getObjectType().getSwiftRValueType();
+  std::string funcName = mangler.mangleOutlinedAssignWithTakeFunction(canType);
+  auto GenFunc = [this](const TypeInfo &objectTI, IRGenFunction &IGF,
+                        Address dest, Address src, SILType T) {
+    objectTI.assignWithTake(IGF, dest, src, T);
+  };
+  return getOrCreateOutlinedCopyAddrHelperFunction(objectTI, llvmType, addrTy,
+                                                   funcName, GenFunc);
+}
+
+llvm::Constant *IRGenModule::getOrCreateOutlinedAssignWithCopyFunction(
+    const TypeInfo &objectTI, llvm::Type *llvmType, SILType addrTy) {
+  IRGenMangler mangler;
+  CanType canType = addrTy.getObjectType().getSwiftRValueType();
+  std::string funcName = mangler.mangleOutlinedAssignWithCopyFunction(canType);
+  auto GenFunc = [this](const TypeInfo &objectTI, IRGenFunction &IGF,
+                        Address dest, Address src, SILType T) {
+    objectTI.assignWithCopy(IGF, dest, src, T);
+  };
+  return getOrCreateOutlinedCopyAddrHelperFunction(objectTI, llvmType, addrTy,
+                                                   funcName, GenFunc);
+}

lib/IRGen/GenEnum.cpp

@@ -2598,25 +2598,57 @@ namespace {
     void assignWithCopy(IRGenFunction &IGF, Address dest, Address src,
                         SILType T)
     const override {
-      emitIndirectAssign(IGF, dest, src, T, IsNotTake);
+      if (IGF.isInOutlinedFunction() || (TIK < Loadable) ||
+          (T.hasArchetype())) {
+        emitIndirectAssign(IGF, dest, src, T, IsNotTake);
+      } else {
+        // Create an outlined function to avoid explosion
+        IGF.IGM.generateCallToOutlinedCopyAddr(
+            IGF, *TI, dest, src, T,
+            &IRGenModule::getOrCreateOutlinedAssignWithCopyFunction);
+      }
     }
 
     void assignWithTake(IRGenFunction &IGF, Address dest, Address src,
                         SILType T)
     const override {
-      emitIndirectAssign(IGF, dest, src, T, IsTake);
+      if (IGF.isInOutlinedFunction() || (TIK < Loadable) ||
+          (T.hasArchetype())) {
+        emitIndirectAssign(IGF, dest, src, T, IsTake);
+      } else {
+        // Create an outlined function to avoid explosion
+        IGF.IGM.generateCallToOutlinedCopyAddr(
+            IGF, *TI, dest, src, T,
+            &IRGenModule::getOrCreateOutlinedAssignWithTakeFunction);
+      }
     }
 
     void initializeWithCopy(IRGenFunction &IGF, Address dest, Address src,
                             SILType T)
     const override {
-      emitIndirectInitialize(IGF, dest, src, T, IsNotTake);
+      if (IGF.isInOutlinedFunction() || (TIK < Loadable) ||
+          (T.hasArchetype())) {
+        emitIndirectInitialize(IGF, dest, src, T, IsNotTake);
+      } else {
+        // Create an outlined function to avoid explosion
+        IGF.IGM.generateCallToOutlinedCopyAddr(
+            IGF, *TI, dest, src, T,
+            &IRGenModule::getOrCreateOutlinedInitializeWithCopyFunction);
+      }
     }
 
     void initializeWithTake(IRGenFunction &IGF, Address dest, Address src,
                             SILType T)
     const override {
-      emitIndirectInitialize(IGF, dest, src, T, IsTake);
+      if (IGF.isInOutlinedFunction() || (TIK < Loadable) ||
+          (T.hasArchetype())) {
+        emitIndirectInitialize(IGF, dest, src, T, IsTake);
+      } else {
+        // Create an outlined function to avoid explosion
+        IGF.IGM.generateCallToOutlinedCopyAddr(
+            IGF, *TI, dest, src, T,
+            &IRGenModule::getOrCreateOutlinedInitializeWithTakeFunction);
+      }
     }
 
     void storeTag(IRGenFunction &IGF,
@@ -4211,25 +4243,57 @@ namespace {
     void assignWithCopy(IRGenFunction &IGF, Address dest, Address src,
                         SILType T)
     const override {
-      emitIndirectAssign(IGF, dest, src, T, IsNotTake);
+      if (IGF.isInOutlinedFunction() || (TIK < Loadable) ||
+          (T.hasArchetype())) {
+        emitIndirectAssign(IGF, dest, src, T, IsNotTake);
+      } else {
+        // Create an outlined function to avoid explosion
+        IGF.IGM.generateCallToOutlinedCopyAddr(
+            IGF, *TI, dest, src, T,
+            &IRGenModule::getOrCreateOutlinedAssignWithCopyFunction);
+      }
     }
 
     void assignWithTake(IRGenFunction &IGF, Address dest, Address src,
                         SILType T)
     const override {
-      emitIndirectAssign(IGF, dest, src, T, IsTake);
+      if (IGF.isInOutlinedFunction() || (TIK < Loadable) ||
+          (T.hasArchetype())) {
+        emitIndirectAssign(IGF, dest, src, T, IsTake);
+      } else {
+        // Create an outlined function to avoid explosion
+        IGF.IGM.generateCallToOutlinedCopyAddr(
+            IGF, *TI, dest, src, T,
+            &IRGenModule::getOrCreateOutlinedAssignWithTakeFunction);
+      }
     }
 
     void initializeWithCopy(IRGenFunction &IGF, Address dest, Address src,
                             SILType T)
     const override {
-      emitIndirectInitialize(IGF, dest, src, T, IsNotTake);
+      if (IGF.isInOutlinedFunction() || (TIK < Loadable) ||
+          (T.hasArchetype())) {
+        emitIndirectInitialize(IGF, dest, src, T, IsNotTake);
+      } else {
+        // Create an outlined function to avoid explosion
+        IGF.IGM.generateCallToOutlinedCopyAddr(
+            IGF, *TI, dest, src, T,
+            &IRGenModule::getOrCreateOutlinedInitializeWithCopyFunction);
+      }
     }
 
     void initializeWithTake(IRGenFunction &IGF, Address dest, Address src,
                             SILType T)
     const override {
-      emitIndirectInitialize(IGF, dest, src, T, IsTake);
+      if (IGF.isInOutlinedFunction() || (TIK < Loadable) ||
+          (T.hasArchetype())) {
+        emitIndirectInitialize(IGF, dest, src, T, IsTake);
+      } else {
+        // Create an outlined function to avoid explosion
+        IGF.IGM.generateCallToOutlinedCopyAddr(
+            IGF, *TI, dest, src, T,
+            &IRGenModule::getOrCreateOutlinedInitializeWithTakeFunction);
+      }
     }
 
     void destroy(IRGenFunction &IGF, Address addr, SILType T) const override {