Skip to content

Commit adec755

Browse files
gottesmmgottesmm
authored
Merge pull request #32756 from gottesmm/eager_specializer_fix
[ownership] Fix the eager specializer for ownership
2 parents b4ef3da + ac3109a commit adec755

36 files changed

+1458
-94
lines changed

‎docs/SIL.rst

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5148,6 +5148,21 @@ unchecked_bitwise_cast
51485148
Bitwise copies an object of type ``A`` into a new object of type ``B``
51495149
of the same size or smaller.
51505150

5151+
unchecked_value_cast
5152+
````````````````````
5153+
::
5154+
5155+
sil-instruction ::= 'unchecked_value_cast' sil-operand 'to' sil-type
5156+
5157+
%1 = unchecked_value_cast %0 : $A to $B
5158+
5159+
Bitwise copies an object of type ``A`` into a new layout-compatible object of
5160+
type ``B`` of the same size.
5161+
5162+
This instruction is assumed to forward a fixed ownership (set upon its
5163+
construction) and lowers to 'unchecked_bitwise_cast' in non-ossa code. This
5164+
causes the cast to lose its guarantee of layout-compatibility.
5165+
51515166
ref_to_raw_pointer
51525167
``````````````````
51535168
::

‎include/swift/SIL/SILBuilder.h

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1102,6 +1102,12 @@ class SILBuilder {
11021102
getSILDebugLocation(Loc), Op, Ty, getFunction(), C.OpenedArchetypes));
11031103
}
11041104

1105+
UncheckedValueCastInst *createUncheckedValueCast(SILLocation Loc, SILValue Op,
1106+
SILType Ty) {
1107+
return insert(UncheckedValueCastInst::create(
1108+
getSILDebugLocation(Loc), Op, Ty, getFunction(), C.OpenedArchetypes));
1109+
}
1110+
11051111
RefToBridgeObjectInst *createRefToBridgeObject(SILLocation Loc, SILValue Ref,
11061112
SILValue Bits) {
11071113
auto Ty = SILType::getBridgeObjectType(getASTContext());
@@ -1847,7 +1853,18 @@ class SILBuilder {
18471853
// Unchecked cast helpers
18481854
//===--------------------------------------------------------------------===//
18491855

1850-
// Create the appropriate cast instruction based on result type.
1856+
/// Create the appropriate cast instruction based on result type.
1857+
///
1858+
/// NOTE: We allow for non-layout compatible casts that shrink the underlying
1859+
/// type we are bit casting!
1860+
SingleValueInstruction *
1861+
createUncheckedReinterpretCast(SILLocation Loc, SILValue Op, SILType Ty);
1862+
1863+
/// Create an appropriate cast instruction based on result type.
1864+
///
1865+
/// NOTE: This assumes that the input and the result cast are layout
1866+
/// compatible. Reduces to createUncheckedReinterpretCast when ownership is
1867+
/// disabled.
18511868
SingleValueInstruction *createUncheckedBitCast(SILLocation Loc, SILValue Op,
18521869
SILType Ty);
18531870

‎include/swift/SIL/SILCloner.h

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1538,6 +1538,16 @@ visitUncheckedBitwiseCastInst(UncheckedBitwiseCastInst *Inst) {
15381538
getOpType(Inst->getType())));
15391539
}
15401540

1541+
template <typename ImplClass>
1542+
void SILCloner<ImplClass>::visitUncheckedValueCastInst(
1543+
UncheckedValueCastInst *Inst) {
1544+
getBuilder().setCurrentDebugScope(getOpScope(Inst->getDebugScope()));
1545+
recordClonedInstruction(Inst, getBuilder().createUncheckedValueCast(
1546+
getOpLocation(Inst->getLoc()),
1547+
getOpValue(Inst->getOperand()),
1548+
getOpType(Inst->getType())));
1549+
}
1550+
15411551
template<typename ImplClass>
15421552
void
15431553
SILCloner<ImplClass>::

‎include/swift/SIL/SILInstruction.h

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4576,6 +4576,23 @@ class UncheckedBitwiseCastInst final
45764576
SILFunction &F, SILOpenedArchetypesState &OpenedArchetypes);
45774577
};
45784578

4579+
/// Bitwise copy a value into another value of the same size.
4580+
class UncheckedValueCastInst final
4581+
: public UnaryInstructionWithTypeDependentOperandsBase<
4582+
SILInstructionKind::UncheckedValueCastInst, UncheckedValueCastInst,
4583+
OwnershipForwardingConversionInst> {
4584+
friend SILBuilder;
4585+
4586+
UncheckedValueCastInst(SILDebugLocation DebugLoc, SILValue Operand,
4587+
ArrayRef<SILValue> TypeDependentOperands, SILType Ty)
4588+
: UnaryInstructionWithTypeDependentOperandsBase(
4589+
DebugLoc, Operand, TypeDependentOperands, Ty,
4590+
Operand.getOwnershipKind()) {}
4591+
static UncheckedValueCastInst *
4592+
create(SILDebugLocation DebugLoc, SILValue Operand, SILType Ty,
4593+
SILFunction &F, SILOpenedArchetypesState &OpenedArchetypes);
4594+
};
4595+
45794596
/// Build a Builtin.BridgeObject from a heap object reference by bitwise-or-ing
45804597
/// in bits from a word.
45814598
class RefToBridgeObjectInst

‎include/swift/SIL/SILNodes.def

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -500,6 +500,8 @@ ABSTRACT_VALUE_AND_INST(SingleValueInstruction, ValueBase, SILInstruction)
500500
ConversionInst, None, DoesNotRelease)
501501
SINGLE_VALUE_INST(UncheckedBitwiseCastInst, unchecked_bitwise_cast,
502502
ConversionInst, None, DoesNotRelease)
503+
SINGLE_VALUE_INST(UncheckedValueCastInst, unchecked_value_cast,
504+
ConversionInst, None, DoesNotRelease)
503505
SINGLE_VALUE_INST(RefToRawPointerInst, ref_to_raw_pointer,
504506
ConversionInst, None, DoesNotRelease)
505507
SINGLE_VALUE_INST(RawPointerToRefInst, raw_pointer_to_ref,

‎include/swift/SILOptimizer/Utils/CFGOptUtils.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,10 @@ bool splitCriticalEdgesFrom(SILBasicBlock *fromBB,
107107
DominanceInfo *domInfo = nullptr,
108108
SILLoopInfo *loopInfo = nullptr);
109109

110+
/// Splits all critical edges that have `toBB` as a destination.
111+
bool splitCriticalEdgesTo(SILBasicBlock *toBB, DominanceInfo *domInfo = nullptr,
112+
SILLoopInfo *loopInfo = nullptr);
113+
110114
/// Splits the edges between two basic blocks.
111115
///
112116
/// Updates dominance information and loop information if not null.

‎lib/IRGen/IRGenSIL.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1026,6 +1026,9 @@ class IRGenSILFunction :
10261026
void visitUncheckedAddrCastInst(UncheckedAddrCastInst *i);
10271027
void visitUncheckedTrivialBitCastInst(UncheckedTrivialBitCastInst *i);
10281028
void visitUncheckedBitwiseCastInst(UncheckedBitwiseCastInst *i);
1029+
void visitUncheckedValueCastInst(UncheckedValueCastInst *i) {
1030+
llvm_unreachable("Should never be seen in Lowered SIL");
1031+
}
10291032
void visitRefToRawPointerInst(RefToRawPointerInst *i);
10301033
void visitRawPointerToRefInst(RawPointerToRefInst *i);
10311034
void visitThinToThickFunctionInst(ThinToThickFunctionInst *i);

‎lib/IRGen/LoadableByAddress.cpp

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1544,7 +1544,7 @@ void LoadableStorageAllocation::
15441544
storageType);
15451545
if (pass.containsDifferentFunctionSignature(pass.F->getLoweredFunctionType(),
15461546
storageType)) {
1547-
auto *castInstr = argBuilder.createUncheckedBitCast(
1547+
auto *castInstr = argBuilder.createUncheckedReinterpretCast(
15481548
RegularLocation(const_cast<ValueDecl *>(arg->getDecl())), arg,
15491549
newSILType);
15501550
arg->replaceAllUsesWith(castInstr);
@@ -1912,8 +1912,8 @@ static void castTupleInstr(SingleValueInstruction *instr, IRGenModule &Mod,
19121912
switch (instr->getKind()) {
19131913
// Add cast to the new sil function type:
19141914
case SILInstructionKind::TupleExtractInst: {
1915-
castInstr = castBuilder.createUncheckedBitCast(instr->getLoc(), instr,
1916-
newSILType.getObjectType());
1915+
castInstr = castBuilder.createUncheckedReinterpretCast(
1916+
instr->getLoc(), instr, newSILType.getObjectType());
19171917
break;
19181918
}
19191919
case SILInstructionKind::TupleElementAddrInst: {
@@ -2471,8 +2471,8 @@ getOperandTypeWithCastIfNecessary(SILInstruction *containingInstr, SILValue op,
24712471
}
24722472
assert(currSILType.isObject() && "Expected an object type");
24732473
if (newSILType != currSILType) {
2474-
auto castInstr = builder.createUncheckedBitCast(containingInstr->getLoc(),
2475-
op, newSILType);
2474+
auto castInstr = builder.createUncheckedReinterpretCast(
2475+
containingInstr->getLoc(), op, newSILType);
24762476
return castInstr;
24772477
}
24782478
}
@@ -2653,8 +2653,8 @@ bool LoadableByAddress::recreateUncheckedEnumDataInstr(
26532653
auto *takeEnum = enumBuilder.createUncheckedEnumData(
26542654
enumInstr->getLoc(), enumInstr->getOperand(), enumInstr->getElement(),
26552655
caseTy);
2656-
newInstr = enumBuilder.createUncheckedBitCast(enumInstr->getLoc(), takeEnum,
2657-
newType);
2656+
newInstr = enumBuilder.createUncheckedReinterpretCast(enumInstr->getLoc(),
2657+
takeEnum, newType);
26582658
} else {
26592659
newInstr = enumBuilder.createUncheckedEnumData(
26602660
enumInstr->getLoc(), enumInstr->getOperand(), enumInstr->getElement(),
@@ -2708,7 +2708,7 @@ bool LoadableByAddress::fixStoreToBlockStorageInstr(
27082708
if (destType.getObjectType() != srcType) {
27092709
// Add cast to destType
27102710
SILBuilderWithScope castBuilder(instr);
2711-
auto *castInstr = castBuilder.createUncheckedBitCast(
2711+
auto *castInstr = castBuilder.createUncheckedReinterpretCast(
27122712
instr->getLoc(), src, destType.getObjectType());
27132713
instr->setOperand(StoreInst::Src, castInstr);
27142714
}

‎lib/SIL/IR/OperandOwnership.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -350,6 +350,7 @@ FORWARD_ANY_OWNERSHIP_INST(DestructureTuple)
350350
FORWARD_ANY_OWNERSHIP_INST(InitExistentialRef)
351351
FORWARD_ANY_OWNERSHIP_INST(DifferentiableFunction)
352352
FORWARD_ANY_OWNERSHIP_INST(LinearFunction)
353+
FORWARD_ANY_OWNERSHIP_INST(UncheckedValueCast)
353354
#undef FORWARD_ANY_OWNERSHIP_INST
354355

355356
// An instruction that forwards a constant ownership or trivial ownership.

‎lib/SIL/IR/SILBuilder.cpp

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,8 @@ SILBuilder::createClassifyBridgeObject(SILLocation Loc, SILValue value) {
141141

142142
// Create the appropriate cast instruction based on result type.
143143
SingleValueInstruction *
144-
SILBuilder::createUncheckedBitCast(SILLocation Loc, SILValue Op, SILType Ty) {
144+
SILBuilder::createUncheckedReinterpretCast(SILLocation Loc, SILValue Op,
145+
SILType Ty) {
145146
assert(isLoadableOrOpaque(Ty));
146147
if (Ty.isTrivial(getFunction()))
147148
return insert(UncheckedTrivialBitCastInst::create(
@@ -156,6 +157,26 @@ SILBuilder::createUncheckedBitCast(SILLocation Loc, SILValue Op, SILType Ty) {
156157
getSILDebugLocation(Loc), Op, Ty, getFunction(), C.OpenedArchetypes));
157158
}
158159

160+
// Create the appropriate cast instruction based on result type.
161+
SingleValueInstruction *
162+
SILBuilder::createUncheckedBitCast(SILLocation Loc, SILValue Op, SILType Ty) {
163+
// Without ownership, delegate to unchecked reinterpret cast.
164+
if (!hasOwnership())
165+
return createUncheckedReinterpretCast(Loc, Op, Ty);
166+
167+
assert(isLoadableOrOpaque(Ty));
168+
if (Ty.isTrivial(getFunction()))
169+
return insert(UncheckedTrivialBitCastInst::create(
170+
getSILDebugLocation(Loc), Op, Ty, getFunction(), C.OpenedArchetypes));
171+
172+
if (SILType::canRefCast(Op->getType(), Ty, getModule()))
173+
return createUncheckedRefCast(Loc, Op, Ty);
174+
175+
// The destination type is nontrivial, and may be smaller than the source
176+
// type, so RC identity cannot be assumed.
177+
return createUncheckedValueCast(Loc, Op, Ty);
178+
}
179+
159180
BranchInst *SILBuilder::createBranch(SILLocation Loc,
160181
SILBasicBlock *TargetBlock,
161182
OperandValueArrayRef Args) {

‎lib/SIL/IR/SILInstructions.cpp

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2217,6 +2217,21 @@ UncheckedRefCastInst::create(SILDebugLocation DebugLoc, SILValue Operand,
22172217
TypeDependentOperands, Ty);
22182218
}
22192219

2220+
UncheckedValueCastInst *
2221+
UncheckedValueCastInst::create(SILDebugLocation DebugLoc, SILValue Operand,
2222+
SILType Ty, SILFunction &F,
2223+
SILOpenedArchetypesState &OpenedArchetypes) {
2224+
SILModule &Mod = F.getModule();
2225+
SmallVector<SILValue, 8> TypeDependentOperands;
2226+
collectTypeDependentOperands(TypeDependentOperands, OpenedArchetypes, F,
2227+
Ty.getASTType());
2228+
unsigned size =
2229+
totalSizeToAlloc<swift::Operand>(1 + TypeDependentOperands.size());
2230+
void *Buffer = Mod.allocateInst(size, alignof(UncheckedValueCastInst));
2231+
return ::new (Buffer)
2232+
UncheckedValueCastInst(DebugLoc, Operand, TypeDependentOperands, Ty);
2233+
}
2234+
22202235
UncheckedAddrCastInst *
22212236
UncheckedAddrCastInst::create(SILDebugLocation DebugLoc, SILValue Operand,
22222237
SILType Ty, SILFunction &F,

‎lib/SIL/IR/SILPrinter.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1620,6 +1620,9 @@ class SILPrinter : public SILInstructionVisitor<SILPrinter> {
16201620
void visitUncheckedBitwiseCastInst(UncheckedBitwiseCastInst *CI) {
16211621
printUncheckedConversionInst(CI, CI->getOperand());
16221622
}
1623+
void visitUncheckedValueCastInst(UncheckedValueCastInst *CI) {
1624+
printUncheckedConversionInst(CI, CI->getOperand());
1625+
}
16231626
void visitRefToRawPointerInst(RefToRawPointerInst *CI) {
16241627
printUncheckedConversionInst(CI, CI->getOperand());
16251628
}

‎lib/SIL/IR/ValueOwnership.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -254,6 +254,7 @@ FORWARDING_OWNERSHIP_INST(Tuple)
254254
FORWARDING_OWNERSHIP_INST(UncheckedRefCast)
255255
FORWARDING_OWNERSHIP_INST(UnconditionalCheckedCast)
256256
FORWARDING_OWNERSHIP_INST(Upcast)
257+
FORWARDING_OWNERSHIP_INST(UncheckedValueCast)
257258
FORWARDING_OWNERSHIP_INST(UncheckedEnumData)
258259
FORWARDING_OWNERSHIP_INST(SelectEnum)
259260
FORWARDING_OWNERSHIP_INST(Enum)

‎lib/SIL/Parser/ParseSIL.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3238,6 +3238,7 @@ bool SILParser::parseSpecificSILInstruction(SILBuilder &B,
32383238
case SILInstructionKind::UncheckedAddrCastInst:
32393239
case SILInstructionKind::UncheckedTrivialBitCastInst:
32403240
case SILInstructionKind::UncheckedBitwiseCastInst:
3241+
case SILInstructionKind::UncheckedValueCastInst:
32413242
case SILInstructionKind::UpcastInst:
32423243
case SILInstructionKind::AddressToPointerInst:
32433244
case SILInstructionKind::BridgeObjectToRefInst:
@@ -3307,6 +3308,9 @@ bool SILParser::parseSpecificSILInstruction(SILBuilder &B,
33073308
case SILInstructionKind::UncheckedBitwiseCastInst:
33083309
ResultVal = B.createUncheckedBitwiseCast(InstLoc, Val, Ty);
33093310
break;
3311+
case SILInstructionKind::UncheckedValueCastInst:
3312+
ResultVal = B.createUncheckedValueCast(InstLoc, Val, Ty);
3313+
break;
33103314
case SILInstructionKind::UpcastInst:
33113315
ResultVal = B.createUpcast(InstLoc, Val, Ty);
33123316
break;

‎lib/SIL/Utils/OwnershipUtils.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ bool swift::isOwnershipForwardingValueKind(SILNodeKind kind) {
3434
case SILNodeKind::LinearFunctionInst:
3535
case SILNodeKind::OpenExistentialRefInst:
3636
case SILNodeKind::UpcastInst:
37+
case SILNodeKind::UncheckedValueCastInst:
3738
case SILNodeKind::UncheckedRefCastInst:
3839
case SILNodeKind::ConvertFunctionInst:
3940
case SILNodeKind::RefToBridgeObjectInst:

‎lib/SIL/Verifier/SILVerifier.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1904,6 +1904,12 @@ class SILVerifier : public SILVerifierBase<SILVerifier> {
19041904
"Inst with qualified ownership in a function that is not qualified");
19051905
}
19061906

1907+
void checkUncheckedValueCastInst(UncheckedValueCastInst *) {
1908+
require(
1909+
F.hasOwnership(),
1910+
"Inst with qualified ownership in a function that is not qualified");
1911+
}
1912+
19071913
template <class AI>
19081914
void checkAccessEnforcement(AI *AccessInst) {
19091915
if (AccessInst->getModule().getStage() != SILStage::Raw) {

‎lib/SILGen/SILGenApply.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1340,8 +1340,8 @@ class SILGenApply : public Lowering::ExprVisitor<SILGenApply> {
13401340
&& loweredResultTy.getASTType()->hasDynamicSelfType()) {
13411341
assert(selfMetaTy);
13421342
selfValue = SGF.emitManagedRValueWithCleanup(
1343-
SGF.B.createUncheckedBitCast(loc, selfValue.forward(SGF),
1344-
loweredResultTy));
1343+
SGF.B.createUncheckedReinterpretCast(loc, selfValue.forward(SGF),
1344+
loweredResultTy));
13451345
} else {
13461346
selfValue = SGF.emitManagedRValueWithCleanup(
13471347
SGF.B.createUpcast(loc, selfValue.forward(SGF), loweredResultTy));

‎lib/SILGen/SILGenBuilder.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -610,7 +610,7 @@ ManagedValue SILGenBuilder::createUncheckedBitCast(SILLocation loc,
610610
ManagedValue value,
611611
SILType type) {
612612
CleanupCloner cloner(*this, value);
613-
SILValue cast = createUncheckedBitCast(loc, value.getValue(), type);
613+
SILValue cast = createUncheckedReinterpretCast(loc, value.getValue(), type);
614614

615615
// Currently createUncheckedBitCast only produces these
616616
// instructions. We assert here to make sure if this changes, this code is

‎lib/SILGen/SILGenBuilder.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -279,7 +279,7 @@ class SILGenBuilder : public SILBuilder {
279279
ManagedValue createUncheckedAddrCast(SILLocation loc, ManagedValue op,
280280
SILType resultTy);
281281

282-
using SILBuilder::createUncheckedBitCast;
282+
using SILBuilder::createUncheckedReinterpretCast;
283283
ManagedValue createUncheckedBitCast(SILLocation loc, ManagedValue original,
284284
SILType type);
285285

‎lib/SILGen/SILGenExpr.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2066,8 +2066,8 @@ RValue RValueEmitter::visitUnderlyingToOpaqueExpr(UnderlyingToOpaqueExpr *E,
20662066
if (value.getType() == opaqueTL.getLoweredType())
20672067
return RValue(SGF, E, value);
20682068

2069-
auto cast = SGF.B.createUncheckedBitCast(E, value.forward(SGF),
2070-
opaqueTL.getLoweredType());
2069+
auto cast = SGF.B.createUncheckedReinterpretCast(E, value.forward(SGF),
2070+
opaqueTL.getLoweredType());
20712071
value = SGF.emitManagedRValueWithCleanup(cast);
20722072

20732073
return RValue(SGF, E, value);

0 commit comments

Comments
 (0)