Skip to content

Commit b3d0b96

Browse files
borsbors
committed
Auto merge of #115764 - RalfJung:const-by-ref-alloc-id, r=<try>
use AllocId instead of Allocation in ConstValue::ByRef This helps avoid redundant AllocIds when a `ByRef` constant gets put back into the interpreter. r? `@oli-obk`
2 parents 366dab1 + 3163962 commit b3d0b96

File tree

43 files changed

+245
-252
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+245
-252
lines changed

‎compiler/rustc_codegen_cranelift/src/constant.rs

Lines changed: 25 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ pub(crate) fn codegen_tls_ref<'tcx>(
7676
pub(crate) fn eval_mir_constant<'tcx>(
7777
fx: &FunctionCx<'_, '_, 'tcx>,
7878
constant: &Constant<'tcx>,
79-
) -> Option<(ConstValue<'tcx>, Ty<'tcx>)> {
79+
) -> Option<(ConstValue, Ty<'tcx>)> {
8080
let constant_kind = fx.monomorphize(constant.literal);
8181
let uv = match constant_kind {
8282
ConstantKind::Ty(const_) => match const_.kind() {
@@ -127,7 +127,7 @@ pub(crate) fn codegen_constant_operand<'tcx>(
127127

128128
pub(crate) fn codegen_const_value<'tcx>(
129129
fx: &mut FunctionCx<'_, '_, 'tcx>,
130-
const_val: ConstValue<'tcx>,
130+
const_val: ConstValue,
131131
ty: Ty<'tcx>,
132132
) -> CValue<'tcx> {
133133
let layout = fx.layout_of(ty);
@@ -222,12 +222,18 @@ pub(crate) fn codegen_const_value<'tcx>(
222222
CValue::by_val(val, layout)
223223
}
224224
},
225-
ConstValue::ByRef { alloc, offset } => CValue::by_ref(
226-
pointer_for_allocation(fx, alloc)
227-
.offset_i64(fx, i64::try_from(offset.bytes()).unwrap()),
228-
layout,
229-
),
230-
ConstValue::Slice { data, start, end } => {
225+
ConstValue::Indirect { alloc_id, offset } => {
226+
let alloc = fx.tcx.global_alloc(alloc_id).unwrap_memory();
227+
// FIXME: avoid creating multiple allocations for the same AllocId?
228+
CValue::by_ref(
229+
pointer_for_allocation(fx, alloc)
230+
.offset_i64(fx, i64::try_from(offset.bytes()).unwrap()),
231+
layout,
232+
)
233+
}
234+
ConstValue::Slice { alloc_id: Some(alloc_id), start, end } => {
235+
let data = fx.tcx.global_alloc(alloc_id).unwrap_memory();
236+
// FIXME: avoid creating multiple allocations for the same AllocId?
231237
let ptr = pointer_for_allocation(fx, data)
232238
.offset_i64(fx, i64::try_from(start).unwrap())
233239
.get_addr(fx);
@@ -237,14 +243,23 @@ pub(crate) fn codegen_const_value<'tcx>(
237243
.iconst(fx.pointer_type, i64::try_from(end.checked_sub(start).unwrap()).unwrap());
238244
CValue::by_val_pair(ptr, len, layout)
239245
}
246+
ConstValue::Slice { alloc_id: None, start, end } => {
247+
assert_eq!(start, end);
248+
// Empty slice at `start`.
249+
// FIXME: `start` could in principle be big enough to overflow into the negative.
250+
// How should the clif value be constructed then?
251+
let addr = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(start).unwrap());
252+
let len = fx.bcx.ins().iconst(fx.pointer_type, 0);
253+
CValue::by_val_pair(addr, len, layout)
254+
}
240255
}
241256
}
242257

243258
fn pointer_for_allocation<'tcx>(
244259
fx: &mut FunctionCx<'_, '_, 'tcx>,
245260
alloc: ConstAllocation<'tcx>,
246261
) -> crate::pointer::Pointer {
247-
let alloc_id = fx.tcx.create_memory_alloc(alloc);
262+
let alloc_id = fx.tcx.reserve_and_set_memory_alloc(alloc);
248263
let data_id = data_id_for_alloc_id(
249264
&mut fx.constants_cx,
250265
&mut *fx.module,
@@ -477,7 +492,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
477492
pub(crate) fn mir_operand_get_const_val<'tcx>(
478493
fx: &FunctionCx<'_, '_, 'tcx>,
479494
operand: &Operand<'tcx>,
480-
) -> Option<ConstValue<'tcx>> {
495+
) -> Option<ConstValue> {
481496
match operand {
482497
Operand::Constant(const_) => Some(eval_mir_constant(fx, const_).unwrap().0),
483498
// FIXME(rust-lang/rust#85105): Casts like `IMM8 as u32` result in the const being stored

‎compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
172172
.expect("simd_shuffle idx not const");
173173

174174
let idx_bytes = match idx_const {
175-
ConstValue::ByRef { alloc, offset } => {
175+
ConstValue::Indirect { alloc_id, offset } => {
176+
let alloc = fx.tcx.global_alloc(alloc_id).unwrap_memory();
176177
let size = Size::from_bytes(
177178
4 * ret_lane_count, /* size_of([u32; ret_lane_count]) */
178179
);

‎compiler/rustc_codegen_ssa/src/common.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,7 @@ pub fn shift_mask_val<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
194194
pub fn asm_const_to_str<'tcx>(
195195
tcx: TyCtxt<'tcx>,
196196
sp: Span,
197-
const_value: ConstValue<'tcx>,
197+
const_value: ConstValue,
198198
ty_and_layout: TyAndLayout<'tcx>,
199199
) -> String {
200200
let ConstValue::Scalar(scalar) = const_value else {

‎compiler/rustc_codegen_ssa/src/mir/constant.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
2323
pub fn eval_mir_constant(
2424
&self,
2525
constant: &mir::Constant<'tcx>,
26-
) -> Result<ConstValue<'tcx>, ErrorHandled> {
26+
) -> Result<ConstValue, ErrorHandled> {
2727
let ct = self.monomorphize(constant.literal);
2828
let uv = match ct {
2929
mir::ConstantKind::Ty(ct) => match ct.kind() {

‎compiler/rustc_codegen_ssa/src/mir/operand.rs

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
8686

8787
pub fn from_const<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
8888
bx: &mut Bx,
89-
val: ConstValue<'tcx>,
89+
val: ConstValue,
9090
ty: Ty<'tcx>,
9191
) -> Self {
9292
let layout = bx.layout_of(ty);
@@ -100,12 +100,12 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
100100
OperandValue::Immediate(llval)
101101
}
102102
ConstValue::ZeroSized => return OperandRef::zero_sized(layout),
103-
ConstValue::Slice { data, start, end } => {
103+
ConstValue::Slice { alloc_id, start, end } => {
104104
let Abi::ScalarPair(a_scalar, _) = layout.abi else {
105105
bug!("from_const: invalid ScalarPair layout: {:#?}", layout);
106106
};
107-
let a = Scalar::from_pointer(
108-
Pointer::new(bx.tcx().create_memory_alloc(data), Size::from_bytes(start)),
107+
let a = Scalar::from_maybe_pointer(
108+
Pointer::new(alloc_id, Size::from_bytes(start)),
109109
&bx.tcx(),
110110
);
111111
let a_llval = bx.scalar_to_backend(
@@ -116,7 +116,9 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
116116
let b_llval = bx.const_usize((end - start) as u64);
117117
OperandValue::Pair(a_llval, b_llval)
118118
}
119-
ConstValue::ByRef { alloc, offset } => {
119+
ConstValue::Indirect { alloc_id, offset } => {
120+
let alloc = bx.tcx().global_alloc(alloc_id).unwrap_memory();
121+
// FIXME: should we attempt to avoid building the same AllocId multiple times?
120122
return Self::from_const_alloc(bx, layout, alloc, offset);
121123
}
122124
};

‎compiler/rustc_const_eval/src/const_eval/eval_queries.rs

Lines changed: 9 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,9 @@ use super::{CanAccessStatics, CompileTimeEvalContext, CompileTimeInterpreter};
1818
use crate::errors;
1919
use crate::interpret::eval_nullary_intrinsic;
2020
use crate::interpret::{
21-
intern_const_alloc_recursive, Allocation, ConstAlloc, ConstValue, CtfeValidationMode, GlobalId,
22-
Immediate, InternKind, InterpCx, InterpError, InterpResult, MPlaceTy, MemoryKind, OpTy,
23-
RefTracking, StackPopCleanup,
21+
intern_const_alloc_recursive, ConstAlloc, ConstValue, CtfeValidationMode, GlobalId, Immediate,
22+
InternKind, InterpCx, InterpError, InterpResult, MPlaceTy, MemoryKind, OpTy, RefTracking,
23+
StackPopCleanup,
2424
};
2525

2626
// Returns a pointer to where the result lives
@@ -111,7 +111,7 @@ pub(super) fn mk_eval_cx<'mir, 'tcx>(
111111
pub(super) fn op_to_const<'tcx>(
112112
ecx: &CompileTimeEvalContext<'_, 'tcx>,
113113
op: &OpTy<'tcx>,
114-
) -> ConstValue<'tcx> {
114+
) -> ConstValue {
115115
// We do not have value optimizations for everything.
116116
// Only scalars and slices, since they are very common.
117117
// Note that further down we turn scalars of uninitialized bits back to `ByRef`. These can result
@@ -148,10 +148,7 @@ pub(super) fn op_to_const<'tcx>(
148148
let to_const_value = |mplace: &MPlaceTy<'_>| {
149149
debug!("to_const_value(mplace: {:?})", mplace);
150150
match mplace.ptr().into_parts() {
151-
(Some(alloc_id), offset) => {
152-
let alloc = ecx.tcx.global_alloc(alloc_id).unwrap_memory();
153-
ConstValue::ByRef { alloc, offset }
154-
}
151+
(Some(alloc_id), offset) => ConstValue::Indirect { alloc_id, offset },
155152
(None, offset) => {
156153
assert!(mplace.layout.is_zst());
157154
assert_eq!(
@@ -173,21 +170,11 @@ pub(super) fn op_to_const<'tcx>(
173170
Immediate::ScalarPair(a, b) => {
174171
debug!("ScalarPair(a: {:?}, b: {:?})", a, b);
175172
// We know `offset` is relative to the allocation, so we can use `into_parts`.
176-
let (data, start) = match a.to_pointer(ecx).unwrap().into_parts() {
177-
(Some(alloc_id), offset) => {
178-
(ecx.tcx.global_alloc(alloc_id).unwrap_memory(), offset.bytes())
179-
}
180-
(None, _offset) => (
181-
ecx.tcx.mk_const_alloc(Allocation::from_bytes_byte_aligned_immutable(
182-
b"" as &[u8],
183-
)),
184-
0,
185-
),
186-
};
173+
let (alloc_id, start) = a.to_pointer(ecx).unwrap().into_parts();
187174
let len = b.to_target_usize(ecx).unwrap();
188-
let start = start.try_into().unwrap();
175+
let start = start.bytes().try_into().unwrap();
189176
let len: usize = len.try_into().unwrap();
190-
ConstValue::Slice { data, start, end: start + len }
177+
ConstValue::Slice { alloc_id, start, end: start + len }
191178
}
192179
Immediate::Uninit => to_const_value(&op.assert_mem_place()),
193180
},
@@ -199,7 +186,7 @@ pub(crate) fn turn_into_const_value<'tcx>(
199186
tcx: TyCtxt<'tcx>,
200187
constant: ConstAlloc<'tcx>,
201188
key: ty::ParamEnvAnd<'tcx, GlobalId<'tcx>>,
202-
) -> ConstValue<'tcx> {
189+
) -> ConstValue {
203190
let cid = key.value;
204191
let def_id = cid.instance.def.def_id();
205192
let is_static = tcx.is_static(def_id);

‎compiler/rustc_const_eval/src/const_eval/mod.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ pub(crate) use valtrees::{const_to_valtree_inner, valtree_to_const_value};
2222
pub(crate) fn const_caller_location(
2323
tcx: TyCtxt<'_>,
2424
(file, line, col): (Symbol, u32, u32),
25-
) -> ConstValue<'_> {
25+
) -> ConstValue {
2626
trace!("const_caller_location: {}:{}:{}", file, line, col);
2727
let mut ecx = mk_eval_cx(tcx, DUMMY_SP, ty::ParamEnv::reveal_all(), CanAccessStatics::No);
2828

@@ -87,7 +87,7 @@ pub(crate) fn eval_to_valtree<'tcx>(
8787
#[instrument(skip(tcx), level = "debug")]
8888
pub(crate) fn try_destructure_mir_constant_for_diagnostics<'tcx>(
8989
tcx: TyCtxt<'tcx>,
90-
val: ConstValue<'tcx>,
90+
val: ConstValue,
9191
ty: Ty<'tcx>,
9292
) -> Option<mir::DestructuredConstant<'tcx>> {
9393
let param_env = ty::ParamEnv::reveal_all();

‎compiler/rustc_const_eval/src/const_eval/valtrees.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,7 @@ pub fn valtree_to_const_value<'tcx>(
207207
tcx: TyCtxt<'tcx>,
208208
param_env_ty: ty::ParamEnvAnd<'tcx, Ty<'tcx>>,
209209
valtree: ty::ValTree<'tcx>,
210-
) -> ConstValue<'tcx> {
210+
) -> ConstValue {
211211
// Basic idea: We directly construct `Scalar` values from trivial `ValTree`s
212212
// (those for constants with type bool, int, uint, float or char).
213213
// For all other types we create an `MPlace` and fill that by walking

‎compiler/rustc_const_eval/src/interpret/cast.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
8484
)
8585
.ok_or_else(|| err_inval!(TooGeneric))?;
8686

87-
let fn_ptr = self.create_fn_alloc_ptr(FnVal::Instance(instance));
87+
let fn_ptr = self.fn_ptr(FnVal::Instance(instance));
8888
self.write_pointer(fn_ptr, dest)?;
8989
}
9090
_ => span_bug!(self.cur_span(), "reify fn pointer on {:?}", src.layout.ty),
@@ -116,7 +116,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
116116
ty::ClosureKind::FnOnce,
117117
)
118118
.ok_or_else(|| err_inval!(TooGeneric))?;
119-
let fn_ptr = self.create_fn_alloc_ptr(FnVal::Instance(instance));
119+
let fn_ptr = self.fn_ptr(FnVal::Instance(instance));
120120
self.write_pointer(fn_ptr, dest)?;
121121
}
122122
_ => span_bug!(self.cur_span(), "closure fn pointer on {:?}", src.layout.ty),

‎compiler/rustc_const_eval/src/interpret/intern.rs

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@ use rustc_middle::ty::{self, layout::TyAndLayout, Ty};
2424
use rustc_ast::Mutability;
2525

2626
use super::{
27-
AllocId, Allocation, ConstAllocation, InterpCx, MPlaceTy, Machine, MemoryKind, PlaceTy,
28-
Projectable, ValueVisitor,
27+
AllocId, Allocation, InterpCx, MPlaceTy, Machine, MemoryKind, PlaceTy, Projectable,
28+
ValueVisitor,
2929
};
3030
use crate::const_eval;
3131
use crate::errors::{DanglingPtrInFinal, UnsupportedUntypedPointer};
@@ -455,19 +455,23 @@ impl<'mir, 'tcx: 'mir, M: super::intern::CompileTimeMachine<'mir, 'tcx, !>>
455455
{
456456
/// A helper function that allocates memory for the layout given and gives you access to mutate
457457
/// it. Once your own mutation code is done, the backing `Allocation` is removed from the
458-
/// current `Memory` and returned.
458+
/// current `Memory` and interned as read-only into the global memory.
459459
pub fn intern_with_temp_alloc(
460460
&mut self,
461461
layout: TyAndLayout<'tcx>,
462462
f: impl FnOnce(
463463
&mut InterpCx<'mir, 'tcx, M>,
464464
&PlaceTy<'tcx, M::Provenance>,
465465
) -> InterpResult<'tcx, ()>,
466-
) -> InterpResult<'tcx, ConstAllocation<'tcx>> {
466+
) -> InterpResult<'tcx, AllocId> {
467+
// `allocate` picks a fresh AllocId that we will associate with its data below.
467468
let dest = self.allocate(layout, MemoryKind::Stack)?;
468469
f(self, &dest.clone().into())?;
469470
let mut alloc = self.memory.alloc_map.remove(&dest.ptr().provenance.unwrap()).unwrap().1;
470471
alloc.mutability = Mutability::Not;
471-
Ok(self.tcx.mk_const_alloc(alloc))
472+
let alloc = self.tcx.mk_const_alloc(alloc);
473+
let alloc_id = dest.ptr().provenance.unwrap(); // this was just allocated, it must have provenance
474+
self.tcx.set_alloc_id_memory(alloc_id, alloc);
475+
Ok(alloc_id)
472476
}
473477
}

‎compiler/rustc_const_eval/src/interpret/intrinsics.rs

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,7 @@
55
use rustc_hir::def_id::DefId;
66
use rustc_middle::mir::{
77
self,
8-
interpret::{
9-
Allocation, ConstAllocation, ConstValue, GlobalId, InterpResult, PointerArithmetic, Scalar,
10-
},
8+
interpret::{Allocation, ConstValue, GlobalId, InterpResult, PointerArithmetic, Scalar},
119
BinOp, NonDivergingIntrinsic,
1210
};
1311
use rustc_middle::ty;
@@ -43,28 +41,24 @@ fn numeric_intrinsic<Prov>(name: Symbol, bits: u128, kind: Primitive) -> Scalar<
4341
Scalar::from_uint(bits_out, size)
4442
}
4543

46-
/// Directly returns an `Allocation` containing an absolute path representation of the given type.
47-
pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ConstAllocation<'tcx> {
48-
let path = crate::util::type_name(tcx, ty);
49-
let alloc = Allocation::from_bytes_byte_aligned_immutable(path.into_bytes());
50-
tcx.mk_const_alloc(alloc)
51-
}
52-
5344
/// The logic for all nullary intrinsics is implemented here. These intrinsics don't get evaluated
5445
/// inside an `InterpCx` and instead have their value computed directly from rustc internal info.
5546
pub(crate) fn eval_nullary_intrinsic<'tcx>(
5647
tcx: TyCtxt<'tcx>,
5748
param_env: ty::ParamEnv<'tcx>,
5849
def_id: DefId,
5950
args: GenericArgsRef<'tcx>,
60-
) -> InterpResult<'tcx, ConstValue<'tcx>> {
51+
) -> InterpResult<'tcx, ConstValue> {
6152
let tp_ty = args.type_at(0);
6253
let name = tcx.item_name(def_id);
6354
Ok(match name {
6455
sym::type_name => {
6556
ensure_monomorphic_enough(tcx, tp_ty)?;
66-
let alloc = alloc_type_name(tcx, tp_ty);
67-
ConstValue::Slice { data: alloc, start: 0, end: alloc.inner().len() }
57+
let path = crate::util::type_name(tcx, tp_ty);
58+
let alloc = Allocation::from_bytes_byte_aligned_immutable(path.into_bytes());
59+
let alloc = tcx.mk_const_alloc(alloc);
60+
let alloc_id = tcx.reserve_and_set_memory_alloc(alloc);
61+
ConstValue::Slice { alloc_id: Some(alloc_id), start: 0, end: alloc.inner().len() }
6862
}
6963
sym::needs_drop => {
7064
ensure_monomorphic_enough(tcx, tp_ty)?;

‎compiler/rustc_const_eval/src/interpret/machine.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -552,7 +552,7 @@ pub macro compile_time_machine(<$mir: lifetime, $tcx: lifetime>) {
552552
def_id: DefId,
553553
) -> InterpResult<$tcx, Pointer> {
554554
// Use the `AllocId` associated with the `DefId`. Any actual *access* will fail.
555-
Ok(Pointer::new(ecx.tcx.create_static_alloc(def_id), Size::ZERO))
555+
Ok(Pointer::new(ecx.tcx.reserve_and_set_static_alloc(def_id), Size::ZERO))
556556
}
557557

558558
#[inline(always)]

‎compiler/rustc_const_eval/src/interpret/memory.rs

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -176,12 +176,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
176176
M::adjust_alloc_base_pointer(self, ptr)
177177
}
178178

179-
pub fn create_fn_alloc_ptr(
180-
&mut self,
181-
fn_val: FnVal<'tcx, M::ExtraFnVal>,
182-
) -> Pointer<M::Provenance> {
179+
pub fn fn_ptr(&mut self, fn_val: FnVal<'tcx, M::ExtraFnVal>) -> Pointer<M::Provenance> {
183180
let id = match fn_val {
184-
FnVal::Instance(instance) => self.tcx.create_fn_alloc(instance),
181+
FnVal::Instance(instance) => self.tcx.reserve_and_set_fn_alloc(instance),
185182
FnVal::Other(extra) => {
186183
// FIXME(RalfJung): Should we have a cache here?
187184
let id = self.tcx.reserve_alloc_id();

0 commit comments

Comments
 (0)