Skip to content

Commit 4d35ddf

Browse files
ZoxcZoxc
committed
Tweak query code for performance
1 parent 58e9832 commit 4d35ddf

File tree

8 files changed

+108
-46
lines changed

8 files changed

+108
-46
lines changed

‎src/librustc/dep_graph/dep_node.rs

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,9 @@ macro_rules! define_dep_nodes {
162162
}
163163
}
164164

165-
#[inline]
165+
// FIXME: Make `is_anon`, `is_input`, `is_eval_always` and `has_params` properties
166+
// of queries
167+
#[inline(always)]
166168
pub fn is_anon(&self) -> bool {
167169
match *self {
168170
$(
@@ -171,16 +173,20 @@ macro_rules! define_dep_nodes {
171173
}
172174
}
173175

174-
#[inline]
175-
pub fn is_input(&self) -> bool {
176+
#[inline(always)]
177+
pub fn is_input_inlined(&self) -> bool {
176178
match *self {
177179
$(
178180
DepKind :: $variant => { contains_input_attr!($($attr),*) }
179181
)*
180182
}
181183
}
182184

183-
#[inline]
185+
pub fn is_input(&self) -> bool {
186+
self.is_input_inlined()
187+
}
188+
189+
#[inline(always)]
184190
pub fn is_eval_always(&self) -> bool {
185191
match *self {
186192
$(

‎src/librustc/dep_graph/graph.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,7 @@ impl DepGraph {
239239
arg: A,
240240
no_tcx: bool,
241241
task: fn(C, A) -> R,
242+
// FIXME: Take OpenTask as a parameter instead
242243
create_task: fn(DepNode) -> OpenTask,
243244
finish_task_and_alloc_depnode: fn(&Lock<CurrentDepGraph>,
244245
DepNode,

‎src/librustc/lib.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@
6060
#![feature(slice_sort_by_cached_key)]
6161
#![feature(specialization)]
6262
#![feature(unboxed_closures)]
63+
#![feature(thread_local)]
6364
#![feature(trace_macros)]
6465
#![feature(trusted_len)]
6566
#![feature(vec_remove_item)]

‎src/librustc/session/mod.rs

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -825,13 +825,22 @@ impl Session {
825825
}
826826
}
827827

828-
pub fn profiler<F: FnOnce(&mut SelfProfiler) -> ()>(&self, f: F) {
828+
#[inline(never)]
829+
#[cold]
830+
fn profiler_active<F: FnOnce(&mut SelfProfiler) -> ()>(&self, f: F) {
829831
if self.opts.debugging_opts.self_profile {
830832
let mut profiler = self.self_profiling.borrow_mut();
831833
f(&mut profiler);
832834
}
833835
}
834836

837+
#[inline(always)]
838+
pub fn profiler<F: FnOnce(&mut SelfProfiler) -> ()>(&self, f: F) {
839+
if unsafe { std::intrinsics::unlikely(self.opts.debugging_opts.self_profile) } {
840+
self.profiler_active(f)
841+
}
842+
}
843+
835844
pub fn print_profiler_results(&self) {
836845
let mut profiler = self.self_profiling.borrow_mut();
837846
profiler.print_results(&self.opts);

‎src/librustc/ty/context.rs

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1926,23 +1926,33 @@ pub mod tls {
19261926

19271927
/// A thread local variable which stores a pointer to the current ImplicitCtxt
19281928
#[cfg(not(parallel_queries))]
1929-
thread_local!(static TLV: Cell<usize> = Cell::new(0));
1929+
// Accessing `thread_local` in another crate is bugged, so we have
1930+
// two accessors `set_raw_tlv` and `get_tlv` which do not have an
1931+
// inline attribute to prevent that
1932+
#[thread_local]
1933+
static TLV: Cell<usize> = Cell::new(0);
1934+
1935+
/// This is used to set the pointer to the current ImplicitCtxt.
1936+
#[cfg(not(parallel_queries))]
1937+
fn set_raw_tlv(value: usize) {
1938+
TLV.set(value)
1939+
}
19301940

19311941
/// Sets TLV to `value` during the call to `f`.
19321942
/// It is restored to its previous value after.
19331943
/// This is used to set the pointer to the new ImplicitCtxt.
19341944
#[cfg(not(parallel_queries))]
19351945
fn set_tlv<F: FnOnce() -> R, R>(value: usize, f: F) -> R {
19361946
let old = get_tlv();
1937-
let _reset = OnDrop(move || TLV.with(|tlv| tlv.set(old)));
1938-
TLV.with(|tlv| tlv.set(value));
1947+
let _reset = OnDrop(move || set_raw_tlv(old));
1948+
set_raw_tlv(value);
19391949
f()
19401950
}
19411951

19421952
/// This is used to get the pointer to the current ImplicitCtxt.
19431953
#[cfg(not(parallel_queries))]
19441954
fn get_tlv() -> usize {
1945-
TLV.with(|tlv| tlv.get())
1955+
TLV.get()
19461956
}
19471957

19481958
/// This is a callback from libsyntax as it cannot access the implicit state

‎src/librustc/ty/query/job.rs

Lines changed: 37 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,11 @@ use syntax_pos::Span;
1818
use ty::tls;
1919
use ty::query::Query;
2020
use ty::query::plumbing::CycleError;
21+
#[cfg(not(parallel_queries))]
22+
use ty::query::{
23+
plumbing::TryGetJob,
24+
config::QueryDescription,
25+
};
2126
use ty::context::TyCtxt;
2227
use errors::Diagnostic;
2328
use std::process;
@@ -83,41 +88,49 @@ impl<'tcx> QueryJob<'tcx> {
8388
///
8489
/// For single threaded rustc there's no concurrent jobs running, so if we are waiting for any
8590
/// query that means that there is a query cycle, thus this always running a cycle error.
91+
#[cfg(not(parallel_queries))]
92+
#[inline(never)]
93+
#[cold]
94+
pub(super) fn await<'lcx, 'a, D: QueryDescription<'tcx>>(
95+
&self,
96+
tcx: TyCtxt<'_, 'tcx, 'lcx>,
97+
span: Span,
98+
) -> TryGetJob<'a, 'tcx, D> {
99+
TryGetJob::JobCompleted(Err(self.find_cycle_in_stack(tcx, span)))
100+
}
101+
102+
/// Awaits for the query job to complete.
103+
///
104+
/// For single threaded rustc there's no concurrent jobs running, so if we are waiting for any
105+
/// query that means that there is a query cycle, thus this always running a cycle error.
106+
#[cfg(parallel_queries)]
86107
pub(super) fn await<'lcx>(
87108
&self,
88109
tcx: TyCtxt<'_, 'tcx, 'lcx>,
89110
span: Span,
90111
) -> Result<(), CycleError<'tcx>> {
91-
#[cfg(not(parallel_queries))]
92-
{
93-
self.find_cycle_in_stack(tcx, span)
94-
}
95-
96-
#[cfg(parallel_queries)]
97-
{
98-
tls::with_related_context(tcx, move |icx| {
99-
let mut waiter = Lrc::new(QueryWaiter {
100-
query: icx.query.clone(),
101-
span,
102-
cycle: Lock::new(None),
103-
condvar: Condvar::new(),
104-
});
105-
self.latch.await(&waiter);
106-
107-
match Lrc::get_mut(&mut waiter).unwrap().cycle.get_mut().take() {
108-
None => Ok(()),
109-
Some(cycle) => Err(cycle)
110-
}
111-
})
112-
}
112+
tls::with_related_context(tcx, move |icx| {
113+
let mut waiter = Lrc::new(QueryWaiter {
114+
query: icx.query.clone(),
115+
span,
116+
cycle: Lock::new(None),
117+
condvar: Condvar::new(),
118+
});
119+
self.latch.await(&waiter);
120+
121+
match Lrc::get_mut(&mut waiter).unwrap().cycle.get_mut().take() {
122+
None => Ok(()),
123+
Some(cycle) => Err(cycle)
124+
}
125+
})
113126
}
114127

115128
#[cfg(not(parallel_queries))]
116129
fn find_cycle_in_stack<'lcx>(
117130
&self,
118131
tcx: TyCtxt<'_, 'tcx, 'lcx>,
119132
span: Span,
120-
) -> Result<(), CycleError<'tcx>> {
133+
) -> CycleError<'tcx> {
121134
// Get the current executing query (waiter) and find the waitee amongst its parents
122135
let mut current_job = tls::with_related_context(tcx, |icx| icx.query.clone());
123136
let mut cycle = Vec::new();
@@ -137,7 +150,7 @@ impl<'tcx> QueryJob<'tcx> {
137150
let usage = job.parent.as_ref().map(|parent| {
138151
(job.info.span, parent.info.query.clone())
139152
});
140-
return Err(CycleError { usage, cycle });
153+
return CycleError { usage, cycle };
141154
}
142155

143156
current_job = job.parent.clone();

‎src/librustc/ty/query/mod.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -705,21 +705,21 @@ impl<'a, 'tcx, 'lcx> TyCtxt<'a, 'tcx, 'lcx> {
705705
self,
706706
span: Span,
707707
key: DefId,
708-
) -> Result<&'tcx [Ty<'tcx>], DiagnosticBuilder<'a>> {
708+
) -> Result<&'tcx [Ty<'tcx>], Box<DiagnosticBuilder<'a>>> {
709709
self.try_get_query::<queries::adt_sized_constraint<'_>>(span, key)
710710
}
711711
pub fn try_needs_drop_raw(
712712
self,
713713
span: Span,
714714
key: ty::ParamEnvAnd<'tcx, Ty<'tcx>>,
715-
) -> Result<bool, DiagnosticBuilder<'a>> {
715+
) -> Result<bool, Box<DiagnosticBuilder<'a>>> {
716716
self.try_get_query::<queries::needs_drop_raw<'_>>(span, key)
717717
}
718718
pub fn try_optimized_mir(
719719
self,
720720
span: Span,
721721
key: DefId,
722-
) -> Result<&'tcx mir::Mir<'tcx>, DiagnosticBuilder<'a>> {
722+
) -> Result<&'tcx mir::Mir<'tcx>, Box<DiagnosticBuilder<'a>>> {
723723
self.try_get_query::<queries::optimized_mir<'_>>(span, key)
724724
}
725725
}

‎src/librustc/ty/query/plumbing.rs

Lines changed: 32 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -153,8 +153,14 @@ impl<'a, 'tcx, Q: QueryDescription<'tcx>> JobOwner<'a, 'tcx, Q> {
153153
};
154154
mem::drop(lock);
155155

156-
if let Err(cycle) = job.await(tcx, span) {
157-
return TryGetJob::JobCompleted(Err(cycle));
156+
#[cfg(not(parallel_queries))]
157+
return job.await(tcx, span);
158+
159+
#[cfg(parallel_queries)]
160+
{
161+
if let Err(cycle) = job.await(tcx, span) {
162+
return TryGetJob::JobCompleted(Err(cycle));
163+
}
158164
}
159165
}
160166
}
@@ -245,8 +251,10 @@ pub(super) enum TryGetJob<'a, 'tcx: 'a, D: QueryDescription<'tcx> + 'a> {
245251
}
246252

247253
impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
254+
#[inline(never)]
255+
#[cold]
248256
pub(super) fn report_cycle(self, CycleError { usage, cycle: stack }: CycleError<'gcx>)
249-
-> DiagnosticBuilder<'a>
257+
-> Box<DiagnosticBuilder<'a>>
250258
{
251259
assert!(!stack.is_empty());
252260

@@ -280,7 +288,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
280288
&format!("cycle used when {}", query.describe(self)));
281289
}
282290

283-
return err
291+
return Box::new(err)
284292
})
285293
}
286294

@@ -345,6 +353,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
345353
}
346354
}
347355

356+
#[inline(never)]
348357
fn try_get_with<Q: QueryDescription<'gcx>>(
349358
self,
350359
span: Span,
@@ -409,7 +418,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
409418
return Ok(result);
410419
}
411420

412-
if !dep_node.kind.is_input() {
421+
if !dep_node.kind.is_input_inlined() {
413422
if let Some(dep_node_index) = self.try_mark_green_and_read(&dep_node) {
414423
profq_msg!(self, ProfileQueriesMsg::CacheHit);
415424
self.sess.profiler(|p| p.record_query_hit(Q::CATEGORY));
@@ -585,7 +594,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
585594

586595
// Ensuring an "input" or anonymous query makes no sense
587596
assert!(!dep_node.kind.is_anon());
588-
assert!(!dep_node.kind.is_input());
597+
assert!(!dep_node.kind.is_input_inlined());
589598
if self.try_mark_green_and_read(&dep_node).is_none() {
590599
// A None return from `try_mark_green_and_read` means that this is either
591600
// a new dep node or that the dep node has already been marked red.
@@ -625,7 +634,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
625634
self,
626635
span: Span,
627636
key: Q::Key,
628-
) -> Result<Q::Value, DiagnosticBuilder<'a>> {
637+
) -> Result<Q::Value, Box<DiagnosticBuilder<'a>>> {
629638
match self.try_get_with::<Q>(span, key) {
630639
Ok(e) => Ok(e),
631640
Err(e) => Err(self.report_cycle(e)),
@@ -637,11 +646,20 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
637646
span: Span,
638647
key: Q::Key,
639648
) -> Q::Value {
640-
self.try_get_query::<Q>(span, key).unwrap_or_else(|mut e| {
641-
e.emit();
642-
Q::handle_cycle_error(self)
649+
self.try_get_query::<Q>(span, key).unwrap_or_else(|e| {
650+
self.emit_error::<Q>(e)
643651
})
644652
}
653+
654+
#[inline(never)]
655+
#[cold]
656+
fn emit_error<Q: QueryDescription<'gcx>>(
657+
self,
658+
mut e: Box<DiagnosticBuilder<'a>>,
659+
) -> Q::Value {
660+
e.emit();
661+
Q::handle_cycle_error(self)
662+
}
645663
}
646664

647665
macro_rules! handle_cycle_error {
@@ -861,6 +879,7 @@ macro_rules! define_queries_inner {
861879

862880
impl<'a, 'gcx, 'tcx> Deref for TyCtxtAt<'a, 'gcx, 'tcx> {
863881
type Target = TyCtxt<'a, 'gcx, 'tcx>;
882+
#[inline(always)]
864883
fn deref(&self) -> &Self::Target {
865884
&self.tcx
866885
}
@@ -869,6 +888,7 @@ macro_rules! define_queries_inner {
869888
impl<'a, $tcx, 'lcx> TyCtxt<'a, $tcx, 'lcx> {
870889
/// Return a transparent wrapper for `TyCtxt` which uses
871890
/// `span` as the location of queries performed through it.
891+
#[inline(always)]
872892
pub fn at(self, span: Span) -> TyCtxtAt<'a, $tcx, 'lcx> {
873893
TyCtxtAt {
874894
tcx: self,
@@ -877,13 +897,15 @@ macro_rules! define_queries_inner {
877897
}
878898

879899
$($(#[$attr])*
900+
#[inline(always)]
880901
pub fn $name(self, key: $K) -> $V {
881902
self.at(DUMMY_SP).$name(key)
882903
})*
883904
}
884905

885906
impl<'a, $tcx, 'lcx> TyCtxtAt<'a, $tcx, 'lcx> {
886907
$($(#[$attr])*
908+
#[inline(always)]
887909
pub fn $name(self, key: $K) -> $V {
888910
self.tcx.get_query::<queries::$name<'_>>(self.span, key)
889911
})*

0 commit comments

Comments
 (0)