IRGen: don't use value witness functions in outlined value operations for functions with performance constraints

SwiftCompilerSources/Sources/Optimizer/ModulePasses/MandatoryPerformanceOptimizations.swift

@@ -50,6 +50,12 @@ private func optimizeFunctionsTopDown(using worklist: inout FunctionWorklist,
         return
       }
 
+      // It's not required to set the perf_constraint flag on all functions in embedded mode.
+      // Embedded mode already implies that flag.
+      if !moduleContext.options.enableEmbeddedSwift {
+        f.set(isPerformanceConstraint: true, context)
+      }
+
       optimize(function: f, context, &worklist)
     }
 

SwiftCompilerSources/Sources/Optimizer/PassManager/Context.swift

@@ -620,6 +620,12 @@ extension Function {
     }
   }
 
+  func set(isPerformanceConstraint: Bool, _ context: FunctionPassContext) {
+    context.notifyEffectsChanged()
+    bridged.setIsPerformanceConstraint(isPerformanceConstraint)
+  }
+
+
   func fixStackNesting(_ context: FunctionPassContext) {
     context._bridged.fixStackNesting(bridged)
   }

docs/ABI/Mangling.rst

@@ -328,9 +328,13 @@ with a differentiable function used for differentiable programming.
   global ::= generic-signature? type 'WOs' // Outlined release
   global ::= generic-signature? type 'WOb' // Outlined initializeWithTake
   global ::= generic-signature? type 'WOc' // Outlined initializeWithCopy
+  global ::= generic-signature? type 'WOC' // Outlined initializeWithCopy, not using value witness
   global ::= generic-signature? type 'WOd' // Outlined assignWithTake
+  global ::= generic-signature? type 'WOD' // Outlined assignWithTake, not using value witness
   global ::= generic-signature? type 'WOf' // Outlined assignWithCopy
+  global ::= generic-signature? type 'WOF' // Outlined assignWithCopy, not using value witness
   global ::= generic-signature? type 'WOh' // Outlined destroy
+  global ::= generic-signature? type 'WOH' // Outlined destroy, not using value witness
   global ::= generic-signature? type 'WOi` // Outlined store enum tag
   global ::= generic-signature? type 'WOj` // Outlined enum destructive project
   global ::= generic-signature? type 'WOg` // Outlined enum get tag

docs/SIL.rst

@@ -1280,7 +1280,14 @@ The clang node owner.
 
 Specifies the performance constraints for the function, which defines which type
 of runtime functions are allowed to be called from the function.
+::
+
+  sil-function-attribute ::= '[perf_constraint]'
 
+Specifies that the optimizer and IRGen must not add runtime calls which are not
+in the function originally. This attribute is set for functions with performance
+constraints or functions which are called from functions with performance
+constraints.
 
 Argument Effects
 ````````````````

include/swift/Demangling/DemangleNodes.def

@@ -381,5 +381,10 @@ NODE(AsyncRemoved)
 // Added in Swift 5.TBD
 NODE(ObjectiveCProtocolSymbolicReference)
 
+NODE(OutlinedInitializeWithCopyNoValueWitness)
+NODE(OutlinedAssignWithTakeNoValueWitness)
+NODE(OutlinedAssignWithCopyNoValueWitness)
+NODE(OutlinedDestroyNoValueWitness)
+
 #undef CONTEXT_NODE
 #undef NODE

include/swift/SIL/SILBridging.h

@@ -542,6 +542,7 @@ struct BridgedFunction {
   BRIDGED_INLINE void setThunk(ThunkKind) const;
   BRIDGED_INLINE bool needsStackProtection() const;
   BRIDGED_INLINE void setNeedStackProtection(bool needSP) const;
+  BRIDGED_INLINE void setIsPerformanceConstraint(bool isPerfConstraint) const;
   BRIDGED_INLINE bool isResilientNominalDecl(BridgedNominalTypeDecl decl) const;
   BRIDGED_INLINE BridgedType getLoweredType(BridgedASTType type) const;
 

include/swift/SIL/SILBridgingImpl.h

@@ -672,6 +672,10 @@ void BridgedFunction::setNeedStackProtection(bool needSP) const {
   getFunction()->setNeedStackProtection(needSP);
 }
 
+void BridgedFunction::setIsPerformanceConstraint(bool isPerfConstraint) const {
+  getFunction()->setIsPerformanceConstraint(isPerfConstraint);
+}
+
 bool BridgedFunction::isResilientNominalDecl(BridgedNominalTypeDecl decl) const {
   return decl.unbridged()->isResilient(getFunction()->getModule().getSwiftModule(),
                                        getFunction()->getResilienceExpansion());

include/swift/SIL/SILFunction.h

@@ -445,6 +445,14 @@ class SILFunction
 
   unsigned HasResultDependsOnSelf : 1;
 
+  /// True, if this function or a caller (transitively) has a performance
+  /// constraint.
+  /// If true, optimizations must not introduce new runtime calls or metadata
+  /// creation, which are not there after SILGen.
+  /// Note that this flag is not serialized, because it's computed locally
+  /// within a module by the MandatoryOptimizations pass.
+  unsigned IsPerformanceConstraint : 1;
+
   static void
   validateSubclassScope(SubclassScope scope, IsThunk_t isThunk,
                         const GenericSpecializationInformation *genericInfo) {
@@ -1031,6 +1039,13 @@ class SILFunction
     perfConstraints = perfConstr;
   }
 
+  // see `IsPerformanceConstraint`
+  bool isPerformanceConstraint() const { return IsPerformanceConstraint; }
+
+  void setIsPerformanceConstraint(bool flag = true) {
+    IsPerformanceConstraint = flag;
+  }
+
   /// \returns True if the function is optimizable (i.e. not marked as no-opt),
   ///          or is raw SIL (so that the mandatory passes still run).
   bool shouldOptimize() const;

lib/Demangling/Demangler.cpp

@@ -3427,6 +3427,34 @@ NodePointer Demangler::demangleWitness() {
     }
     case 'O': {
       switch (nextChar()) {
+      case 'C': {
+        if (auto sig = popNode(Node::Kind::DependentGenericSignature))
+          return createWithChildren(Node::Kind::OutlinedInitializeWithCopyNoValueWitness,
+                                    popNode(Node::Kind::Type), sig);
+        return createWithChild(Node::Kind::OutlinedInitializeWithCopyNoValueWitness,
+                               popNode(Node::Kind::Type));
+      }
+      case 'D': {
+        if (auto sig = popNode(Node::Kind::DependentGenericSignature))
+          return createWithChildren(Node::Kind::OutlinedAssignWithTakeNoValueWitness,
+                                    popNode(Node::Kind::Type), sig);
+        return createWithChild(Node::Kind::OutlinedAssignWithTakeNoValueWitness,
+                               popNode(Node::Kind::Type));
+      }
+      case 'F': {
+        if (auto sig = popNode(Node::Kind::DependentGenericSignature))
+          return createWithChildren(Node::Kind::OutlinedAssignWithCopyNoValueWitness,
+                                    popNode(Node::Kind::Type), sig);
+        return createWithChild(Node::Kind::OutlinedAssignWithCopyNoValueWitness,
+                               popNode(Node::Kind::Type));
+      }
+      case 'H': {
+        if (auto sig = popNode(Node::Kind::DependentGenericSignature))
+          return createWithChildren(Node::Kind::OutlinedDestroyNoValueWitness,
+                                    popNode(Node::Kind::Type), sig);
+        return createWithChild(Node::Kind::OutlinedDestroyNoValueWitness,
+                               popNode(Node::Kind::Type));
+      }
       case 'y': {
         if (auto sig = popNode(Node::Kind::DependentGenericSignature))
           return createWithChildren(Node::Kind::OutlinedCopy,

lib/Demangling/NodePrinter.cpp

@@ -573,6 +573,10 @@ class NodePrinter {
     case Node::Kind::OutlinedAssignWithTake:
     case Node::Kind::OutlinedAssignWithCopy:
     case Node::Kind::OutlinedDestroy:
+    case Node::Kind::OutlinedInitializeWithCopyNoValueWitness:
+    case Node::Kind::OutlinedAssignWithTakeNoValueWitness:
+    case Node::Kind::OutlinedAssignWithCopyNoValueWitness:
+    case Node::Kind::OutlinedDestroyNoValueWitness:
     case Node::Kind::OutlinedEnumTagStore:
     case Node::Kind::OutlinedEnumGetTag:
     case Node::Kind::OutlinedEnumProjectDataForLoad:
@@ -1381,18 +1385,22 @@ NodePointer NodePrinter::print(NodePointer Node, unsigned depth,
     print(Node->getChild(0), depth + 1);
     return nullptr;
   case Node::Kind::OutlinedInitializeWithCopy:
+  case Node::Kind::OutlinedInitializeWithCopyNoValueWitness:
     Printer << "outlined init with copy of ";
     print(Node->getChild(0), depth + 1);
     return nullptr;
   case Node::Kind::OutlinedAssignWithTake:
+  case Node::Kind::OutlinedAssignWithTakeNoValueWitness:
     Printer << "outlined assign with take of ";
     print(Node->getChild(0), depth + 1);
     return nullptr;
   case Node::Kind::OutlinedAssignWithCopy:
+  case Node::Kind::OutlinedAssignWithCopyNoValueWitness:
     Printer << "outlined assign with copy of ";
     print(Node->getChild(0), depth + 1);
     return nullptr;
   case Node::Kind::OutlinedDestroy:
+  case Node::Kind::OutlinedDestroyNoValueWitness:
     Printer << "outlined destroy of ";
     print(Node->getChild(0), depth + 1);
     return nullptr;

lib/Demangling/OldRemangler.cpp

@@ -2549,6 +2549,24 @@ ManglingError Remangler::mangleOutlinedDestroy(Node *node, unsigned depth) {
   Buffer << "Wh";
   return mangleSingleChildNode(node, depth + 1);
 }
+ManglingError Remangler::mangleOutlinedInitializeWithCopyNoValueWitness(Node *node,
+                                                                        unsigned depth) {
+  return MANGLING_ERROR(ManglingError::UnsupportedNodeKind, node);
+}
+
+ManglingError Remangler::mangleOutlinedAssignWithTakeNoValueWitness(Node *node,
+                                                                    unsigned depth) {
+  return MANGLING_ERROR(ManglingError::UnsupportedNodeKind, node);
+}
+
+ManglingError Remangler::mangleOutlinedAssignWithCopyNoValueWitness(Node *node,
+                                                                    unsigned depth) {
+  return MANGLING_ERROR(ManglingError::UnsupportedNodeKind, node);
+}
+
+ManglingError Remangler::mangleOutlinedDestroyNoValueWitness(Node *node, unsigned depth) {
+  return MANGLING_ERROR(ManglingError::UnsupportedNodeKind, node);
+}
 ManglingError Remangler::mangleOutlinedEnumTagStore(Node *node, unsigned depth) {
   Buffer << "Wi";
   return mangleSingleChildNode(node, depth + 1);

lib/Demangling/Remangler.cpp

@@ -3315,6 +3315,34 @@ ManglingError Remangler::mangleOutlinedDestroy(Node *node, unsigned depth) {
   Buffer << "WOh";
   return ManglingError::Success;
 }
+
+ManglingError Remangler::mangleOutlinedInitializeWithCopyNoValueWitness(Node *node,
+                                                                        unsigned depth) {
+  RETURN_IF_ERROR(mangleChildNodes(node, depth + 1));
+  Buffer << "WOC";
+  return ManglingError::Success;
+}
+
+ManglingError Remangler::mangleOutlinedAssignWithTakeNoValueWitness(Node *node,
+                                                                    unsigned depth) {
+  RETURN_IF_ERROR(mangleChildNodes(node, depth + 1));
+  Buffer << "WOD";
+  return ManglingError::Success;
+}
+
+ManglingError Remangler::mangleOutlinedAssignWithCopyNoValueWitness(Node *node,
+                                                                    unsigned depth) {
+  RETURN_IF_ERROR(mangleChildNodes(node, depth + 1));
+  Buffer << "WOF";
+  return ManglingError::Success;
+}
+
+ManglingError Remangler::mangleOutlinedDestroyNoValueWitness(Node *node, unsigned depth) {
+  RETURN_IF_ERROR(mangleChildNodes(node, depth + 1));
+  Buffer << "WOH";
+  return ManglingError::Success;
+}
+
 ManglingError Remangler::mangleOutlinedEnumGetTag(Node *node, unsigned depth) {
   RETURN_IF_ERROR(mangleChildNodes(node, depth + 1));
   Buffer << "WOg";

lib/IRGen/GenDecl.cpp

@@ -6216,7 +6216,8 @@ IRGenModule::getOrCreateHelperFunction(StringRef fnName, llvm::Type *resultTy,
                                        ArrayRef<llvm::Type*> paramTys,
                         llvm::function_ref<void(IRGenFunction &IGF)> generate,
                         bool setIsNoInline,
-                        bool forPrologue) {
+                        bool forPrologue,
+                        bool isPerformanceConstraint) {
   llvm::FunctionType *fnTy =
     llvm::FunctionType::get(resultTy, paramTys, false);
 
@@ -6225,7 +6226,7 @@ IRGenModule::getOrCreateHelperFunction(StringRef fnName, llvm::Type *resultTy,
           Module.getOrInsertFunction(fnName, fnTy).getCallee());
 
   if (llvm::Function *def = shouldDefineHelper(*this, fn, setIsNoInline)) {
-    IRGenFunction IGF(*this, def);
+    IRGenFunction IGF(*this, def, isPerformanceConstraint);
     if (DebugInfo && !forPrologue)
       DebugInfo->emitArtificialFunction(IGF, def);
     generate(IGF);

lib/IRGen/IRGenFunction.cpp

@@ -40,12 +40,14 @@ static llvm::cl::opt<bool> EnableTrapDebugInfo(
     llvm::cl::desc("Generate failure-message functions in the debug info"));
 
 IRGenFunction::IRGenFunction(IRGenModule &IGM, llvm::Function *Fn,
+                             bool isPerformanceConstraint,
                              OptimizationMode OptMode,
                              const SILDebugScope *DbgScope,
                              llvm::Optional<SILLocation> DbgLoc)
     : IGM(IGM), Builder(IGM.getLLVMContext(),
                         IGM.DebugInfo && !IGM.Context.LangOpts.DebuggerSupport),
-      OptMode(OptMode), CurFn(Fn), DbgScope(DbgScope) {
+      OptMode(OptMode), isPerformanceConstraint(isPerformanceConstraint),
+      CurFn(Fn), DbgScope(DbgScope) {
 
   // Make sure the instructions in this function are attached its debug scope.
   if (IGM.DebugInfo) {
@@ -83,6 +85,12 @@ bool IRGenFunction::canStackPromotePackMetadata() const {
          !packMetadataStackPromotionDisabled;
 }
 
+bool IRGenFunction::outliningCanCallValueWitnesses() const {
+  if (!IGM.getOptions().UseTypeLayoutValueHandling)
+    return false;
+  return !isPerformanceConstraint && !IGM.Context.LangOpts.hasFeature(Feature::Embedded);
+}
+
 ModuleDecl *IRGenFunction::getSwiftModule() const {
   return IGM.getSwiftModule();
 }

lib/IRGen/IRGenFunction.h

@@ -74,6 +74,7 @@ class IRGenFunction {
   /// If != OptimizationMode::NotSet, the optimization mode specified with an
   /// function attribute.
   OptimizationMode OptMode;
+  bool isPerformanceConstraint;
 
   llvm::Function *CurFn;
   ModuleDecl *getSwiftModule() const;
@@ -82,6 +83,7 @@ class IRGenFunction {
   const IRGenOptions &getOptions() const;
 
   IRGenFunction(IRGenModule &IGM, llvm::Function *fn,
+                bool isPerformanceConstraint = false,
                 OptimizationMode Mode = OptimizationMode::NotSet,
                 const SILDebugScope *DbgScope = nullptr,
                 llvm::Optional<SILLocation> DbgLoc = llvm::None);
@@ -252,6 +254,8 @@ class IRGenFunction {
   /// heapified.
   bool canStackPromotePackMetadata() const;
 
+  bool outliningCanCallValueWitnesses() const;
+
   void setupAsync(unsigned asyncContextIndex);
   bool isAsync() const { return asyncContextLocation.isValid(); }
 

lib/IRGen/IRGenMangler.h

@@ -552,48 +552,53 @@ class IRGenMangler : public Mangle::ASTMangler {
   }
 
   std::string mangleOutlinedInitializeWithTakeFunction(CanType t,
-                                                       CanGenericSignature sig) {
+                                                       CanGenericSignature sig,
+                                                       bool noValueWitness) {
     beginMangling();
     appendType(t, sig);
     if (sig)
       appendGenericSignature(sig);
-    appendOperator("WOb");
+    appendOperator(noValueWitness ? "WOB" : "WOb");
     return finalize();
   }
   std::string mangleOutlinedInitializeWithCopyFunction(CanType t,
-                                                       CanGenericSignature sig) {
+                                                       CanGenericSignature sig,
+                                                       bool noValueWitness) {
     beginMangling();
     appendType(t, sig);
     if (sig)
       appendGenericSignature(sig);
-    appendOperator("WOc");
+    appendOperator(noValueWitness ? "WOC" : "WOc");
     return finalize();
   }
   std::string mangleOutlinedAssignWithTakeFunction(CanType t,
-                                                   CanGenericSignature sig) {
+                                                   CanGenericSignature sig,
+                                                   bool noValueWitness) {
     beginMangling();
     appendType(t, sig);
     if (sig)
       appendGenericSignature(sig);
-    appendOperator("WOd");
+    appendOperator(noValueWitness ? "WOD" : "WOd");
     return finalize();
   }
   std::string mangleOutlinedAssignWithCopyFunction(CanType t,
-                                                   CanGenericSignature sig) {
+                                                   CanGenericSignature sig,
+                                                   bool noValueWitness) {
     beginMangling();
     appendType(t, sig);
     if (sig)
       appendGenericSignature(sig);
-    appendOperator("WOf");
+    appendOperator(noValueWitness ? "WOF" : "WOf");
     return finalize();
   }
   std::string mangleOutlinedDestroyFunction(CanType t,
-                                            CanGenericSignature sig) {
+                                            CanGenericSignature sig,
+                                            bool noValueWitness) {
     beginMangling();
     appendType(t, sig);
     if (sig)
       appendGenericSignature(sig);
-    appendOperator("WOh");
+    appendOperator(noValueWitness ? "WOH" : "WOh");
     return finalize();
   }
 

lib/IRGen/IRGenModule.h

@@ -1175,7 +1175,8 @@ class IRGenModule {
                                             ArrayRef<llvm::Type*> paramTypes,
                         llvm::function_ref<void(IRGenFunction &IGF)> generate,
                         bool setIsNoInline = false,
-                        bool forPrologue = false);
+                        bool forPrologue = false,
+                        bool isPerformanceConstraint = false);
 
   llvm::Constant *getOrCreateRetainFunction(const TypeInfo &objectTI, SILType t,
                               llvm::Type *llvmType, Atomicity atomicity);

lib/IRGen/IRGenSIL.cpp

@@ -1820,6 +1820,7 @@ IRGenSILFunction::IRGenSILFunction(IRGenModule &IGM, SILFunction *f)
     : IRGenFunction(IGM,
                     IGM.getAddrOfSILFunction(f, ForDefinition,
                                              f->isDynamicallyReplaceable()),
+                    f->isPerformanceConstraint(),
                     f->getOptimizationMode(), f->getDebugScope(),
                     f->getLocation()),
       CurSILFn(f) {