Skip to content

Commit 1506b1f

Browse files
author
Markus Westerlind
committed
perf: Reduce snapshot/rollback overhead
By merging the undo_log of all structures part of the snapshot the cost of creating a snapshot becomes much cheaper. Since snapshots with no or few changes are so frequent this ends up mattering more than the slight overhead of dispatching on the variants that map to each field.
1 parent 61621e2 commit 1506b1f

File tree

15 files changed

+519
-191
lines changed

15 files changed

+519
-191
lines changed

‎Cargo.lock

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -987,8 +987,7 @@ dependencies = [
987987
[[package]]
988988
name = "ena"
989989
version = "0.13.1"
990-
source = "registry+https://github.com/rust-lang/crates.io-index"
991-
checksum = "8944dc8fa28ce4a38f778bd46bf7d923fe73eed5a439398507246c8e017e6f36"
990+
source = "git+https://github.com/Marwes/ena?branch=detach_undo_log#9b977ea7f209a35f46d65d33cdd74b8f4931fb8a"
992991
dependencies = [
993992
"log",
994993
]

‎Cargo.toml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,5 +65,7 @@ rustc-std-workspace-core = { path = 'src/tools/rustc-std-workspace-core' }
6565
rustc-std-workspace-alloc = { path = 'src/tools/rustc-std-workspace-alloc' }
6666
rustc-std-workspace-std = { path = 'src/tools/rustc-std-workspace-std' }
6767

68+
ena = { version = "0.13.1", git = "https://github.com/Marwes/ena", branch = "detach_undo_log" }
69+
6870
[patch."https://github.com/rust-lang/rust-clippy"]
6971
clippy_lints = { path = "src/tools/clippy/clippy_lints" }

‎src/librustc_data_structures/lib.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ pub mod sync;
8484
pub mod thin_vec;
8585
pub mod tiny_list;
8686
pub mod transitive_relation;
87+
pub use ena::undo_log;
8788
pub use ena::unify;
8889
mod atomic_ref;
8990
pub mod fingerprint;

‎src/librustc_infer/infer/combine.rs

Lines changed: 24 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ impl<'infcx, 'tcx> InferCtxt<'infcx, 'tcx> {
7676
(&ty::Infer(ty::IntVar(a_id)), &ty::Infer(ty::IntVar(b_id))) => {
7777
self.inner
7878
.borrow_mut()
79-
.int_unification_table
79+
.int_unification_table()
8080
.unify_var_var(a_id, b_id)
8181
.map_err(|e| int_unification_error(a_is_expected, e))?;
8282
Ok(a)
@@ -98,7 +98,7 @@ impl<'infcx, 'tcx> InferCtxt<'infcx, 'tcx> {
9898
(&ty::Infer(ty::FloatVar(a_id)), &ty::Infer(ty::FloatVar(b_id))) => {
9999
self.inner
100100
.borrow_mut()
101-
.float_unification_table
101+
.float_unification_table()
102102
.unify_var_var(a_id, b_id)
103103
.map_err(|e| float_unification_error(relation.a_is_expected(), e))?;
104104
Ok(a)
@@ -133,8 +133,8 @@ impl<'infcx, 'tcx> InferCtxt<'infcx, 'tcx> {
133133
return Ok(a);
134134
}
135135

136-
let a = replace_if_possible(&mut self.inner.borrow_mut().const_unification_table, a);
137-
let b = replace_if_possible(&mut self.inner.borrow_mut().const_unification_table, b);
136+
let a = replace_if_possible(&mut self.inner.borrow_mut().const_unification_table(), a);
137+
let b = replace_if_possible(&mut self.inner.borrow_mut().const_unification_table(), b);
138138

139139
let a_is_expected = relation.a_is_expected();
140140

@@ -145,7 +145,7 @@ impl<'infcx, 'tcx> InferCtxt<'infcx, 'tcx> {
145145
) => {
146146
self.inner
147147
.borrow_mut()
148-
.const_unification_table
148+
.const_unification_table()
149149
.unify_var_var(a_vid, b_vid)
150150
.map_err(|e| const_unification_error(a_is_expected, e))?;
151151
return Ok(a);
@@ -179,7 +179,7 @@ impl<'infcx, 'tcx> InferCtxt<'infcx, 'tcx> {
179179
) -> RelateResult<'tcx, &'tcx ty::Const<'tcx>> {
180180
self.inner
181181
.borrow_mut()
182-
.const_unification_table
182+
.const_unification_table()
183183
.unify_var_value(
184184
vid,
185185
ConstVarValue {
@@ -202,7 +202,7 @@ impl<'infcx, 'tcx> InferCtxt<'infcx, 'tcx> {
202202
) -> RelateResult<'tcx, Ty<'tcx>> {
203203
self.inner
204204
.borrow_mut()
205-
.int_unification_table
205+
.int_unification_table()
206206
.unify_var_value(vid, Some(val))
207207
.map_err(|e| int_unification_error(vid_is_expected, e))?;
208208
match val {
@@ -219,7 +219,7 @@ impl<'infcx, 'tcx> InferCtxt<'infcx, 'tcx> {
219219
) -> RelateResult<'tcx, Ty<'tcx>> {
220220
self.inner
221221
.borrow_mut()
222-
.float_unification_table
222+
.float_unification_table()
223223
.unify_var_value(vid, Some(ty::FloatVarValue(val)))
224224
.map_err(|e| float_unification_error(vid_is_expected, e))?;
225225
Ok(self.tcx.mk_mach_float(val))
@@ -266,7 +266,7 @@ impl<'infcx, 'tcx> CombineFields<'infcx, 'tcx> {
266266
use self::RelationDir::*;
267267

268268
// Get the actual variable that b_vid has been inferred to
269-
debug_assert!(self.infcx.inner.borrow_mut().type_variables.probe(b_vid).is_unknown());
269+
debug_assert!(self.infcx.inner.borrow_mut().type_variables().probe(b_vid).is_unknown());
270270

271271
debug!("instantiate(a_ty={:?} dir={:?} b_vid={:?})", a_ty, dir, b_vid);
272272

@@ -286,7 +286,7 @@ impl<'infcx, 'tcx> CombineFields<'infcx, 'tcx> {
286286
"instantiate(a_ty={:?}, dir={:?}, b_vid={:?}, generalized b_ty={:?})",
287287
a_ty, dir, b_vid, b_ty
288288
);
289-
self.infcx.inner.borrow_mut().type_variables.instantiate(b_vid, b_ty);
289+
self.infcx.inner.borrow_mut().type_variables().instantiate(b_vid, b_ty);
290290

291291
if needs_wf {
292292
self.obligations.push(Obligation::new(
@@ -344,7 +344,7 @@ impl<'infcx, 'tcx> CombineFields<'infcx, 'tcx> {
344344

345345
debug!("generalize: ambient_variance = {:?}", ambient_variance);
346346

347-
let for_universe = match self.infcx.inner.borrow_mut().type_variables.probe(for_vid) {
347+
let for_universe = match self.infcx.inner.borrow_mut().type_variables().probe(for_vid) {
348348
v @ TypeVariableValue::Known { .. } => {
349349
panic!("instantiating {:?} which has a known value {:?}", for_vid, v,)
350350
}
@@ -356,7 +356,7 @@ impl<'infcx, 'tcx> CombineFields<'infcx, 'tcx> {
356356
let mut generalize = Generalizer {
357357
infcx: self.infcx,
358358
span: self.trace.cause.span,
359-
for_vid_sub_root: self.infcx.inner.borrow_mut().type_variables.sub_root_var(for_vid),
359+
for_vid_sub_root: self.infcx.inner.borrow_mut().type_variables().sub_root_var(for_vid),
360360
for_universe,
361361
ambient_variance,
362362
needs_wf: false,
@@ -508,14 +508,14 @@ impl TypeRelation<'tcx> for Generalizer<'_, 'tcx> {
508508
// us from creating infinitely sized types.
509509
match t.kind {
510510
ty::Infer(ty::TyVar(vid)) => {
511-
let vid = self.infcx.inner.borrow_mut().type_variables.root_var(vid);
512-
let sub_vid = self.infcx.inner.borrow_mut().type_variables.sub_root_var(vid);
511+
let vid = self.infcx.inner.borrow_mut().type_variables().root_var(vid);
512+
let sub_vid = self.infcx.inner.borrow_mut().type_variables().sub_root_var(vid);
513513
if sub_vid == self.for_vid_sub_root {
514514
// If sub-roots are equal, then `for_vid` and
515515
// `vid` are related via subtyping.
516516
Err(TypeError::CyclicTy(self.root_ty))
517517
} else {
518-
let probe = self.infcx.inner.borrow_mut().type_variables.probe(vid);
518+
let probe = self.infcx.inner.borrow_mut().type_variables().probe(vid);
519519
match probe {
520520
TypeVariableValue::Known { value: u } => {
521521
debug!("generalize: known value {:?}", u);
@@ -542,12 +542,13 @@ impl TypeRelation<'tcx> for Generalizer<'_, 'tcx> {
542542
}
543543

544544
let origin =
545-
*self.infcx.inner.borrow_mut().type_variables.var_origin(vid);
546-
let new_var_id = self.infcx.inner.borrow_mut().type_variables.new_var(
547-
self.for_universe,
548-
false,
549-
origin,
550-
);
545+
*self.infcx.inner.borrow_mut().type_variables().var_origin(vid);
546+
let new_var_id = self
547+
.infcx
548+
.inner
549+
.borrow_mut()
550+
.type_variables()
551+
.new_var(self.for_universe, false, origin);
551552
let u = self.tcx().mk_ty_var(new_var_id);
552553
debug!("generalize: replacing original vid={:?} with new={:?}", vid, u);
553554
Ok(u)
@@ -618,7 +619,8 @@ impl TypeRelation<'tcx> for Generalizer<'_, 'tcx> {
618619

619620
match c.val {
620621
ty::ConstKind::Infer(InferConst::Var(vid)) => {
621-
let variable_table = &mut self.infcx.inner.borrow_mut().const_unification_table;
622+
let mut inner = self.infcx.inner.borrow_mut();
623+
let variable_table = &mut inner.const_unification_table();
622624
let var_value = variable_table.probe_value(vid);
623625
match var_value.val {
624626
ConstVariableValue::Known { value: u } => self.relate(&u, &u),

‎src/librustc_infer/infer/equate.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -72,14 +72,14 @@ impl TypeRelation<'tcx> for Equate<'combine, 'infcx, 'tcx> {
7272
}
7373

7474
let infcx = self.fields.infcx;
75-
let a = infcx.inner.borrow_mut().type_variables.replace_if_possible(a);
76-
let b = infcx.inner.borrow_mut().type_variables.replace_if_possible(b);
75+
let a = infcx.inner.borrow_mut().type_variables().replace_if_possible(a);
76+
let b = infcx.inner.borrow_mut().type_variables().replace_if_possible(b);
7777

7878
debug!("{}.tys: replacements ({:?}, {:?})", self.tag(), a, b);
7979

8080
match (&a.kind, &b.kind) {
8181
(&ty::Infer(TyVar(a_id)), &ty::Infer(TyVar(b_id))) => {
82-
infcx.inner.borrow_mut().type_variables.equate(a_id, b_id);
82+
infcx.inner.borrow_mut().type_variables().equate(a_id, b_id);
8383
}
8484

8585
(&ty::Infer(TyVar(a_id)), _) => {

‎src/librustc_infer/infer/error_reporting/need_type_info.rs

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ impl<'a, 'tcx> FindHirNodeVisitor<'a, 'tcx> {
5959
.infcx
6060
.inner
6161
.borrow_mut()
62-
.type_variables
62+
.type_variables()
6363
.sub_unified(a_vid, b_vid),
6464
_ => false,
6565
}
@@ -194,7 +194,8 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> {
194194
highlight: Option<ty::print::RegionHighlightMode>,
195195
) -> (String, Option<Span>, Cow<'static, str>, Option<String>, Option<&'static str>) {
196196
if let ty::Infer(ty::TyVar(ty_vid)) = ty.kind {
197-
let ty_vars = &self.inner.borrow().type_variables;
197+
let mut inner = self.inner.borrow_mut();
198+
let ty_vars = &inner.type_variables();
198199
let var_origin = ty_vars.var_origin(ty_vid);
199200
if let TypeVariableOriginKind::TypeParameterDefinition(name, def_id) = var_origin.kind {
200201
let parent_def_id = def_id.and_then(|def_id| self.tcx.parent(def_id));
@@ -248,7 +249,8 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> {
248249
let ty_to_string = |ty: Ty<'tcx>| -> String {
249250
let mut s = String::new();
250251
let mut printer = ty::print::FmtPrinter::new(self.tcx, &mut s, Namespace::TypeNS);
251-
let ty_vars = &self.inner.borrow().type_variables;
252+
let mut inner = self.inner.borrow_mut();
253+
let ty_vars = inner.type_variables();
252254
let getter = move |ty_vid| {
253255
let var_origin = ty_vars.var_origin(ty_vid);
254256
if let TypeVariableOriginKind::TypeParameterDefinition(name, _) = var_origin.kind {

‎src/librustc_infer/infer/freshen.rs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -147,15 +147,15 @@ impl<'a, 'tcx> TypeFolder<'tcx> for TypeFreshener<'a, 'tcx> {
147147

148148
match t.kind {
149149
ty::Infer(ty::TyVar(v)) => {
150-
let opt_ty = self.infcx.inner.borrow_mut().type_variables.probe(v).known();
150+
let opt_ty = self.infcx.inner.borrow_mut().type_variables().probe(v).known();
151151
self.freshen_ty(opt_ty, ty::TyVar(v), ty::FreshTy)
152152
}
153153

154154
ty::Infer(ty::IntVar(v)) => self.freshen_ty(
155155
self.infcx
156156
.inner
157157
.borrow_mut()
158-
.int_unification_table
158+
.int_unification_table()
159159
.probe_value(v)
160160
.map(|v| v.to_type(tcx)),
161161
ty::IntVar(v),
@@ -166,7 +166,7 @@ impl<'a, 'tcx> TypeFolder<'tcx> for TypeFreshener<'a, 'tcx> {
166166
self.infcx
167167
.inner
168168
.borrow_mut()
169-
.float_unification_table
169+
.float_unification_table()
170170
.probe_value(v)
171171
.map(|v| v.to_type(tcx)),
172172
ty::FloatVar(v),
@@ -222,7 +222,7 @@ impl<'a, 'tcx> TypeFolder<'tcx> for TypeFreshener<'a, 'tcx> {
222222
.infcx
223223
.inner
224224
.borrow_mut()
225-
.const_unification_table
225+
.const_unification_table()
226226
.probe_value(v)
227227
.val
228228
.known();

‎src/librustc_infer/infer/fudge.rs

Lines changed: 28 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,30 @@ use rustc_middle::ty::{self, ConstVid, FloatVid, IntVid, RegionVid, Ty, TyCtxt,
33

44
use super::type_variable::TypeVariableOrigin;
55
use super::InferCtxt;
6-
use super::{ConstVariableOrigin, RegionVariableOrigin};
6+
use super::{ConstVariableOrigin, RegionVariableOrigin, UnificationTable};
77

8+
use rustc_data_structures::snapshot_vec as sv;
89
use rustc_data_structures::unify as ut;
910
use ut::UnifyKey;
1011

1112
use std::ops::Range;
1213

14+
fn vars_since_snapshot<'tcx, T>(
15+
table: &mut UnificationTable<'_, 'tcx, T>,
16+
snapshot: usize,
17+
) -> Range<T>
18+
where
19+
T: UnifyKey,
20+
super::UndoLog<'tcx>: From<sv::UndoLog<ut::Delegate<T>>>,
21+
{
22+
T::from_index(snapshot as u32)..T::from_index(table.len() as u32)
23+
}
24+
1325
fn const_vars_since_snapshot<'tcx>(
14-
table: &mut ut::UnificationTable<ut::InPlace<ConstVid<'tcx>>>,
15-
snapshot: &ut::Snapshot<ut::InPlace<ConstVid<'tcx>>>,
26+
table: &mut UnificationTable<'_, 'tcx, ConstVid<'tcx>>,
27+
snapshot: usize,
1628
) -> (Range<ConstVid<'tcx>>, Vec<ConstVariableOrigin>) {
17-
let range = table.vars_since_snapshot(snapshot);
29+
let range = vars_since_snapshot(table, snapshot);
1830
(
1931
range.start..range.end,
2032
(range.start.index..range.end.index)
@@ -83,17 +95,21 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> {
8395

8496
let mut inner = self.inner.borrow_mut();
8597
let type_vars =
86-
inner.type_variables.vars_since_snapshot(&snapshot.type_snapshot);
87-
let int_vars =
88-
inner.int_unification_table.vars_since_snapshot(&snapshot.int_snapshot);
89-
let float_vars =
90-
inner.float_unification_table.vars_since_snapshot(&snapshot.float_snapshot);
98+
inner.type_variables().vars_since_snapshot(&snapshot.type_snapshot);
99+
let int_vars = vars_since_snapshot(
100+
&mut inner.int_unification_table(),
101+
snapshot.int_snapshot,
102+
);
103+
let float_vars = vars_since_snapshot(
104+
&mut inner.float_unification_table(),
105+
snapshot.float_snapshot,
106+
);
91107
let region_vars = inner
92108
.unwrap_region_constraints()
93109
.vars_since_snapshot(&snapshot.region_constraints_snapshot);
94110
let const_vars = const_vars_since_snapshot(
95-
&mut inner.const_unification_table,
96-
&snapshot.const_snapshot,
111+
&mut inner.const_unification_table(),
112+
snapshot.const_snapshot,
97113
);
98114

99115
let fudger = InferenceFudger {
@@ -161,7 +177,7 @@ impl<'a, 'tcx> TypeFolder<'tcx> for InferenceFudger<'a, 'tcx> {
161177
// that it is unbound, so we can just return
162178
// it.
163179
debug_assert!(
164-
self.infcx.inner.borrow_mut().type_variables.probe(vid).is_unknown()
180+
self.infcx.inner.borrow_mut().type_variables().probe(vid).is_unknown()
165181
);
166182
ty
167183
}

‎src/librustc_infer/infer/lattice.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,8 @@ where
5656
}
5757

5858
let infcx = this.infcx();
59-
let a = infcx.inner.borrow_mut().type_variables.replace_if_possible(a);
60-
let b = infcx.inner.borrow_mut().type_variables.replace_if_possible(b);
59+
let a = infcx.inner.borrow_mut().type_variables().replace_if_possible(a);
60+
let b = infcx.inner.borrow_mut().type_variables().replace_if_possible(b);
6161
match (&a.kind, &b.kind) {
6262
// If one side is known to be a variable and one is not,
6363
// create a variable (`v`) to represent the LUB. Make sure to

0 commit comments

Comments
 (0)