[sil] Add a new instruction called explicit_copy_addr.

This is exactly like copy_addr except that it is not viewed from the verifiers
perspective as an "invalid" copy of a move only value. It is intended to be used
in two contexts:

1. When the move checker emits a diagnostic since it could not eliminate a copy,
we still need to produce valid SIL without copy_addr on move only types since we
will hit canonical SIL eventually even if we don't actually codegen the SIL. The
pass can just convert said copy_addr to explicit_copy_addr and everyone is
happy.

2. To implement the explicit copy function for address only types.

Author: gottesmm

docs/SIL.rst

@@ -4245,6 +4245,25 @@ operations::
 If ``T`` is a trivial type, then ``copy_addr`` is always equivalent to its
 take-initialization form.
 
+It is illegal in non-Raw SIL to apply ``copy_addr [init]`` to a value that is
+move only.
+
+explicit_copy_addr
+``````````````````
+::
+
+  sil-instruction ::= 'explicit_copy_addr' '[take]'? sil-value
+                        'to' '[initialization]'? sil-operand
+
+  explicit_copy_addr [take] %0 to [initialization] %1 : $*T
+  // %0 and %1 must be of the same $*T address type
+
+This instruction is exactly the same as `copy_addr`_ except that it has special
+behavior for move only types. Specifically, an `explicit_copy_addr`_ is viewed
+as a copy_addr that is allowed on values that are move only. This is only used
+by a move checker after it has emitted an error diagnostic to preserve the
+general ``copy_addr [init]`` ban in Canonical SIL on move only types.
+
 destroy_addr
 ````````````
 ::

include/swift/SIL/SILBuilder.h

@@ -1037,6 +1037,14 @@ class SILBuilder {
         getSILDebugLocation(Loc), srcAddr, destAddr, isTake, isInitialize));
   }
 
+  ExplicitCopyAddrInst *
+  createExplicitCopyAddr(SILLocation Loc, SILValue srcAddr, SILValue destAddr,
+                         IsTake_t isTake, IsInitialization_t isInitialize) {
+    assert(srcAddr->getType() == destAddr->getType());
+    return insert(new (getModule()) ExplicitCopyAddrInst(
+        getSILDebugLocation(Loc), srcAddr, destAddr, isTake, isInitialize));
+  }
+
   BindMemoryInst *createBindMemory(SILLocation Loc, SILValue base,
                                    SILValue index, SILType boundType) {
     return insert(BindMemoryInst::create(getSILDebugLocation(Loc), base, index,

include/swift/SIL/SILCloner.h

@@ -1383,6 +1383,17 @@ SILCloner<ImplClass>::visitCopyAddrInst(CopyAddrInst *Inst) {
                 Inst->isInitializationOfDest()));
 }
 
+template <typename ImplClass>
+void SILCloner<ImplClass>::visitExplicitCopyAddrInst(
+    ExplicitCopyAddrInst *Inst) {
+  getBuilder().setCurrentDebugScope(getOpScope(Inst->getDebugScope()));
+  recordClonedInstruction(
+      Inst, getBuilder().createExplicitCopyAddr(
+                getOpLocation(Inst->getLoc()), getOpValue(Inst->getSrc()),
+                getOpValue(Inst->getDest()), Inst->isTakeOfSrc(),
+                Inst->isInitializationOfDest()));
+}
+
 template <typename ImplClass>
 void SILCloner<ImplClass>::visitMarkUnresolvedMoveAddrInst(
     MarkUnresolvedMoveAddrInst *Inst) {

include/swift/SIL/SILInstruction.h

@@ -5000,6 +5000,48 @@ class CopyAddrInst
   MutableArrayRef<Operand> getAllOperands() { return Operands.asArray(); }
 };
 
+/// ExplicitCopyAddrInst - A copy_addr that should not be optimized and should
+/// be viewed
+class ExplicitCopyAddrInst
+    : public InstructionBase<SILInstructionKind::ExplicitCopyAddrInst,
+                             NonValueInstruction>,
+      public CopyLikeInstruction {
+  friend SILBuilder;
+
+private:
+  FixedOperandList<2> Operands;
+  USE_SHARED_UINT8;
+
+  ExplicitCopyAddrInst(SILDebugLocation DebugLoc, SILValue Src, SILValue Dest,
+                       IsTake_t isTakeOfSrc,
+                       IsInitialization_t isInitializationOfDest);
+
+public:
+  SILValue getSrc() const { return Operands[Src].get(); }
+  SILValue getDest() const { return Operands[Dest].get(); }
+
+  void setSrc(SILValue V) { Operands[Src].set(V); }
+  void setDest(SILValue V) { Operands[Dest].set(V); }
+
+  IsTake_t isTakeOfSrc() const {
+    return IsTake_t(sharedUInt8().ExplicitCopyAddrInst.isTakeOfSrc);
+  }
+  IsInitialization_t isInitializationOfDest() const {
+    return IsInitialization_t(
+        sharedUInt8().ExplicitCopyAddrInst.isInitializationOfDest);
+  }
+
+  void setIsTakeOfSrc(IsTake_t T) {
+    sharedUInt8().ExplicitCopyAddrInst.isTakeOfSrc = (bool)T;
+  }
+  void setIsInitializationOfDest(IsInitialization_t I) {
+    sharedUInt8().ExplicitCopyAddrInst.isInitializationOfDest = (bool)I;
+  }
+
+  ArrayRef<Operand> getAllOperands() const { return Operands.asArray(); }
+  MutableArrayRef<Operand> getAllOperands() { return Operands.asArray(); }
+};
+
 /// "%token = bind_memory %0 : $Builtin.RawPointer, %1 : $Builtin.Word to $T"
 ///
 /// Binds memory at the raw pointer %0 to type $T with enough capacity

include/swift/SIL/SILNode.h

@@ -219,6 +219,10 @@ class alignas(8) SILNode :
       isTakeOfSrc : 1,
       isInitializationOfDest : 1);
 
+    SHARED_FIELD(ExplicitCopyAddrInst, uint8_t
+      isTakeOfSrc : 1,
+      isInitializationOfDest : 1);
+
     SHARED_FIELD(PointerToAddressInst, uint8_t
       isStrict : 1,
       isInvariant : 1);

include/swift/SIL/SILNodes.def

@@ -781,6 +781,8 @@ BRIDGED_NON_VALUE_INST(DebugValueInst, debug_value,
 #include "swift/AST/ReferenceStorage.def"
 BRIDGED_NON_VALUE_INST(CopyAddrInst, copy_addr,
                SILInstruction, MayHaveSideEffects, MayRelease)
+BRIDGED_NON_VALUE_INST(ExplicitCopyAddrInst, explicit_copy_addr,
+               SILInstruction, MayHaveSideEffects, MayRelease)
 BRIDGED_NON_VALUE_INST(DestroyAddrInst, destroy_addr,
                SILInstruction, MayHaveSideEffects, MayRelease)
 NON_VALUE_INST(EndLifetimeInst, end_lifetime,

lib/IRGen/IRGenSIL.cpp

@@ -1310,6 +1310,7 @@ class IRGenSILFunction :
   void visitDeallocPartialRefInst(DeallocPartialRefInst *i);
 
   void visitCopyAddrInst(CopyAddrInst *i);
+  void visitExplicitCopyAddrInst(ExplicitCopyAddrInst *i);
   void visitMarkUnresolvedMoveAddrInst(MarkUnresolvedMoveAddrInst *mai) {
     llvm_unreachable("Valid only when ownership is enabled");
   }
@@ -6884,6 +6885,30 @@ void IRGenSILFunction::visitCopyAddrInst(swift::CopyAddrInst *i) {
   }
 }
 
+void IRGenSILFunction::visitExplicitCopyAddrInst(
+    swift::ExplicitCopyAddrInst *i) {
+  SILType addrTy = i->getSrc()->getType();
+  const TypeInfo &addrTI = getTypeInfo(addrTy);
+  Address src = getLoweredAddress(i->getSrc());
+  // See whether we have a deferred fixed-size buffer initialization.
+  auto &loweredDest = getLoweredValue(i->getDest());
+  assert(!loweredDest.isUnallocatedAddressInBuffer());
+  Address dest = loweredDest.getAnyAddress();
+  if (i->isInitializationOfDest()) {
+    if (i->isTakeOfSrc()) {
+      addrTI.initializeWithTake(*this, dest, src, addrTy, false);
+    } else {
+      addrTI.initializeWithCopy(*this, dest, src, addrTy, false);
+    }
+  } else {
+    if (i->isTakeOfSrc()) {
+      addrTI.assignWithTake(*this, dest, src, addrTy, false);
+    } else {
+      addrTI.assignWithCopy(*this, dest, src, addrTy, false);
+    }
+  }
+}
+
 // bind_memory and rebind_memory are no-ops because Swift TBAA info is not
 // lowered to LLVM IR TBAA, and the output token is ignored except for
 // verification.

lib/SIL/IR/OperandOwnership.cpp

@@ -147,6 +147,7 @@ OPERAND_OWNERSHIP(TrivialUse, CheckedCastAddrBranch)
 OPERAND_OWNERSHIP(TrivialUse, CondBranch)
 OPERAND_OWNERSHIP(TrivialUse, CondFail)
 OPERAND_OWNERSHIP(TrivialUse, CopyAddr)
+OPERAND_OWNERSHIP(TrivialUse, ExplicitCopyAddr)
 OPERAND_OWNERSHIP(TrivialUse, MarkUnresolvedMoveAddr)
 OPERAND_OWNERSHIP(TrivialUse, DeallocStack)
 OPERAND_OWNERSHIP(TrivialUse, DeinitExistentialAddr)

lib/SIL/IR/SILInstructions.cpp

@@ -1172,6 +1172,15 @@ CopyAddrInst::CopyAddrInst(SILDebugLocation Loc, SILValue SrcLValue,
       bool(isInitializationOfDest);
   }
 
+  ExplicitCopyAddrInst::ExplicitCopyAddrInst(
+      SILDebugLocation Loc, SILValue SrcLValue, SILValue DestLValue,
+      IsTake_t isTakeOfSrc, IsInitialization_t isInitializationOfDest)
+      : InstructionBase(Loc), Operands(this, SrcLValue, DestLValue) {
+    sharedUInt8().ExplicitCopyAddrInst.isTakeOfSrc = bool(isTakeOfSrc);
+    sharedUInt8().ExplicitCopyAddrInst.isInitializationOfDest =
+        bool(isInitializationOfDest);
+  }
+
 BindMemoryInst *
 BindMemoryInst::create(SILDebugLocation Loc, SILValue Base, SILValue Index,
                        SILType BoundType, SILFunction &F) {

lib/SIL/IR/SILPrinter.cpp

@@ -1749,6 +1749,15 @@ class SILPrinter : public SILInstructionVisitor<SILPrinter> {
     *this << getIDAndType(CI->getDest());
   }
 
+  void visitExplicitCopyAddrInst(ExplicitCopyAddrInst *CI) {
+    if (CI->isTakeOfSrc())
+      *this << "[take] ";
+    *this << Ctx.getID(CI->getSrc()) << " to ";
+    if (CI->isInitializationOfDest())
+      *this << "[initialization] ";
+    *this << getIDAndType(CI->getDest());
+  }
+
   void visitMarkUnresolvedMoveAddrInst(MarkUnresolvedMoveAddrInst *CI) {
     *this << Ctx.getID(CI->getSrc()) << " to ";
     *this << getIDAndType(CI->getDest());

lib/SIL/Parser/ParseSIL.cpp

@@ -4920,6 +4920,37 @@ bool SILParser::parseSpecificSILInstruction(SILBuilder &B,
                                    IsInitialization_t(IsInit));
       break;
     }
+    case SILInstructionKind::ExplicitCopyAddrInst: {
+      bool IsTake = false, IsInit = false;
+      UnresolvedValueName SrcLName;
+      SILValue DestLVal;
+      SourceLoc ToLoc, DestLoc;
+      Identifier ToToken;
+      if (parseSILOptional(IsTake, *this, "take") || parseValueName(SrcLName) ||
+          parseSILIdentifier(ToToken, ToLoc, diag::expected_tok_in_sil_instr,
+                             "to") ||
+          parseSILOptional(IsInit, *this, "initialization") ||
+          parseTypedValueRef(DestLVal, DestLoc, B) ||
+          parseSILDebugLocation(InstLoc, B))
+        return true;
+
+      if (ToToken.str() != "to") {
+        P.diagnose(ToLoc, diag::expected_tok_in_sil_instr, "to");
+        return true;
+      }
+
+      if (!DestLVal->getType().isAddress()) {
+        P.diagnose(DestLoc, diag::sil_invalid_instr_operands);
+        return true;
+      }
+
+      SILValue SrcLVal =
+          getLocalValue(SrcLName, DestLVal->getType(), InstLoc, B);
+      ResultVal =
+          B.createExplicitCopyAddr(InstLoc, SrcLVal, DestLVal, IsTake_t(IsTake),
+                                   IsInitialization_t(IsInit));
+      break;
+    }
     case SILInstructionKind::MarkUnresolvedMoveAddrInst: {
       UnresolvedValueName SrcLName;
       SILValue DestLVal;

lib/SIL/Utils/InstructionUtils.cpp

@@ -630,6 +630,15 @@ RuntimeEffect swift::getRuntimeEffect(SILInstruction *inst, SILType &impactType)
       return RuntimeEffect::MetaData | RuntimeEffect::RefCounting;
     return RuntimeEffect::MetaData;
   }
+  case SILInstructionKind::ExplicitCopyAddrInst: {
+    auto *ca = cast<ExplicitCopyAddrInst>(inst);
+    impactType = ca->getSrc()->getType();
+    if (!ca->isInitializationOfDest())
+      return RuntimeEffect::MetaData | RuntimeEffect::Releasing;
+    if (!ca->isTakeOfSrc())
+      return RuntimeEffect::MetaData | RuntimeEffect::RefCounting;
+    return RuntimeEffect::MetaData;
+  }
   // Equivalent to a copy_addr [init]
   case SILInstructionKind::MarkUnresolvedMoveAddrInst: {
     return RuntimeEffect::MetaData | RuntimeEffect::RefCounting;

lib/SILOptimizer/UtilityPasses/SerializeSILPass.cpp

@@ -314,6 +314,7 @@ static bool hasOpaqueArchetype(TypeExpansionContext context,
   case SILInstructionKind::Store##Name##Inst:
 #include "swift/AST/ReferenceStorage.def"
   case SILInstructionKind::CopyAddrInst:
+  case SILInstructionKind::ExplicitCopyAddrInst:
   case SILInstructionKind::MarkUnresolvedMoveAddrInst:
   case SILInstructionKind::DestroyAddrInst:
   case SILInstructionKind::EndLifetimeInst:

lib/SILOptimizer/Utils/SILInliner.cpp

@@ -884,6 +884,7 @@ InlineCost swift::instructionInlineCost(SILInstruction &I) {
   case SILInstructionKind::CopyBlockInst:
   case SILInstructionKind::CopyBlockWithoutEscapingInst:
   case SILInstructionKind::CopyAddrInst:
+  case SILInstructionKind::ExplicitCopyAddrInst:
   case SILInstructionKind::MarkUnresolvedMoveAddrInst:
   case SILInstructionKind::RetainValueInst:
   case SILInstructionKind::RetainValueAddrInst:

lib/Serialization/DeserializeSIL.cpp

@@ -2171,6 +2171,16 @@ bool SILDeserializer::readSILInstruction(SILFunction *Fn,
         IsTake_t(isTake), IsInitialization_t(isInit));
     break;
   }
+  case SILInstructionKind::ExplicitCopyAddrInst: {
+    auto Ty = MF->getType(TyID);
+    SILType addrType = getSILType(Ty, (SILValueCategory)TyCategory, Fn);
+    bool isInit = (Attr & 0x2) > 0;
+    bool isTake = (Attr & 0x1) > 0;
+    ResultInst = Builder.createExplicitCopyAddr(
+        Loc, getLocalValue(ValID, addrType), getLocalValue(ValID2, addrType),
+        IsTake_t(isTake), IsInitialization_t(isInit));
+    break;
+  }
 
   case SILInstructionKind::MarkUnresolvedMoveAddrInst: {
     auto Ty = MF->getType(TyID);

lib/Serialization/ModuleFormat.h

@@ -58,7 +58,7 @@ const uint16_t SWIFTMODULE_VERSION_MAJOR = 0;
 /// describe what change you made. The content of this comment isn't important;
 /// it just ensures a conflict if two people change the module format.
 /// Don't worry about adhering to the 80-column limit for this line.
-const uint16_t SWIFTMODULE_VERSION_MINOR = 699; // @_alwaysEmitConformanceMetadata
+const uint16_t SWIFTMODULE_VERSION_MINOR = 700; // explicit_copy_addr
 
 /// A standard hash seed used for all string hashes in a serialized module.
 ///

lib/Serialization/SerializeSIL.cpp

@@ -1921,6 +1921,7 @@ void SILSerializer::writeSILInstruction(const SILInstruction &SI) {
 #include "swift/AST/ReferenceStorage.def"
   case SILInstructionKind::AssignInst:
   case SILInstructionKind::CopyAddrInst:
+  case SILInstructionKind::ExplicitCopyAddrInst:
   case SILInstructionKind::MarkUnresolvedMoveAddrInst:
   case SILInstructionKind::StoreInst:
   case SILInstructionKind::StoreBorrowInst: {
@@ -1945,6 +1946,11 @@ void SILSerializer::writeSILInstruction(const SILInstruction &SI) {
       Attr = (CAI->isInitializationOfDest() << 1) | CAI->isTakeOfSrc();
       operand = cast<CopyAddrInst>(&SI)->getDest();
       value = cast<CopyAddrInst>(&SI)->getSrc();
+    } else if (SI.getKind() == SILInstructionKind::ExplicitCopyAddrInst) {
+      const auto *CAI = cast<ExplicitCopyAddrInst>(&SI);
+      Attr = (CAI->isInitializationOfDest() << 1) | CAI->isTakeOfSrc();
+      operand = cast<ExplicitCopyAddrInst>(&SI)->getDest();
+      value = cast<ExplicitCopyAddrInst>(&SI)->getSrc();
     } else if (SI.getKind() == SILInstructionKind::MarkUnresolvedMoveAddrInst) {
       auto *mai = cast<MarkUnresolvedMoveAddrInst>(&SI);
       operand = mai->getDest();

test/SIL/Parser/basic2.sil

@@ -113,3 +113,33 @@ bb0(%0 : @owned $@moveOnly Builtin.NativeObject):
   %9999 = tuple()
   return %9999 : $()
 }
+
+// CHECK-LABEL: sil [ossa] @test_explicit_copy_addr : $@convention(thin) (@owned Builtin.NativeObject) -> () {
+// CHECK: explicit_copy_addr %{{[0-9]+}} to [initialization] %{{[0-9]+}} :
+// CHECK: explicit_copy_addr [take] %{{[0-9]+}} to [initialization] %{{[0-9]+}} :
+// CHECK: explicit_copy_addr [take] %{{[0-9]+}} to %{{[0-9]+}} :
+// CHECK: explicit_copy_addr %{{[0-9]+}} to %{{[0-9]+}} :
+// CHECK: } // end sil function 'test_explicit_copy_addr'
+sil [ossa] @test_explicit_copy_addr : $@convention(thin) (@owned Builtin.NativeObject) -> () {
+bb0(%0 : @owned $Builtin.NativeObject):
+  %1 = alloc_stack $Builtin.NativeObject
+  %2 = alloc_stack $Builtin.NativeObject
+  %3 = alloc_stack $Builtin.NativeObject
+
+  %0a = copy_value %0 : $Builtin.NativeObject
+  store %0a to [init] %1 : $*Builtin.NativeObject
+  explicit_copy_addr %1 to [initialization] %2 : $*Builtin.NativeObject  
+  explicit_copy_addr [take] %2 to [initialization] %3 : $*Builtin.NativeObject
+  explicit_copy_addr [take] %3 to %1 : $*Builtin.NativeObject
+
+  store %0 to [init] %2 : $*Builtin.NativeObject
+  explicit_copy_addr %1 to %2 : $*Builtin.NativeObject
+
+  destroy_addr %2 : $*Builtin.NativeObject
+  destroy_addr %1 : $*Builtin.NativeObject
+  dealloc_stack %3 : $*Builtin.NativeObject
+  dealloc_stack %2 : $*Builtin.NativeObject
+  dealloc_stack %1 : $*Builtin.NativeObject
+  %9999 = tuple()
+  return %9999 : $()
+}

test/SIL/Serialization/basic2.sil

@@ -25,3 +25,33 @@ bb0(%0 : @owned $Builtin.NativeObject):
   %9999 = tuple()
   return %9999 : $()
 }
+
+// CHECK-LABEL: sil [ossa] @test_explicit_copy_addr : $@convention(thin) (@owned Builtin.NativeObject) -> () {
+// CHECK: explicit_copy_addr %{{[0-9]+}} to [initialization] %{{[0-9]+}} :
+// CHECK: explicit_copy_addr [take] %{{[0-9]+}} to [initialization] %{{[0-9]+}} :
+// CHECK: explicit_copy_addr [take] %{{[0-9]+}} to %{{[0-9]+}} :
+// CHECK: explicit_copy_addr %{{[0-9]+}} to %{{[0-9]+}} :
+// CHECK: } // end sil function 'test_explicit_copy_addr'
+sil [ossa] @test_explicit_copy_addr : $@convention(thin) (@owned Builtin.NativeObject) -> () {
+bb0(%0 : @owned $Builtin.NativeObject):
+  %1 = alloc_stack $Builtin.NativeObject
+  %2 = alloc_stack $Builtin.NativeObject
+  %3 = alloc_stack $Builtin.NativeObject
+
+  %0a = copy_value %0 : $Builtin.NativeObject
+  store %0a to [init] %1 : $*Builtin.NativeObject
+  explicit_copy_addr %1 to [initialization] %2 : $*Builtin.NativeObject  
+  explicit_copy_addr [take] %2 to [initialization] %3 : $*Builtin.NativeObject
+  explicit_copy_addr [take] %3 to %1 : $*Builtin.NativeObject
+
+  store %0 to [init] %2 : $*Builtin.NativeObject
+  explicit_copy_addr %1 to %2 : $*Builtin.NativeObject
+
+  destroy_addr %2 : $*Builtin.NativeObject
+  destroy_addr %1 : $*Builtin.NativeObject
+  dealloc_stack %3 : $*Builtin.NativeObject
+  dealloc_stack %2 : $*Builtin.NativeObject
+  dealloc_stack %1 : $*Builtin.NativeObject
+  %9999 = tuple()
+  return %9999 : $()
+}