Skip to content

Commit aed7529

Browse files
KobzolKobzol
committed
Store runtime benchmark compilation errors into the database
1 parent b82ce86 commit aed7529

File tree

4 files changed

+106
-39
lines changed

4 files changed

+106
-39
lines changed

‎collector/src/bin/collector.rs

Lines changed: 47 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ use collector::{runtime, utils, CollectorCtx, CollectorStepBuilder};
1414
use database::{ArtifactId, ArtifactIdNumber, Commit, CommitType, Connection, Pool};
1515
use rayon::iter::{IndexedParallelIterator, IntoParallelRefIterator, ParallelIterator};
1616
use std::cmp::Ordering;
17+
use std::collections::HashMap;
1718
use std::ffi::OsStr;
1819
use std::fs;
1920
use std::fs::File;
@@ -28,8 +29,8 @@ use tokio::runtime::Runtime;
2829
use collector::compile::execute::bencher::BenchProcessor;
2930
use collector::compile::execute::profiler::{ProfileProcessor, Profiler};
3031
use collector::runtime::{
31-
bench_runtime, runtime_benchmark_dir, BenchmarkFilter, BenchmarkSuite, CargoIsolationMode,
32-
DEFAULT_RUNTIME_ITERATIONS,
32+
bench_runtime, runtime_benchmark_dir, BenchmarkFilter, BenchmarkSuite,
33+
BenchmarkSuiteCompilation, CargoIsolationMode, DEFAULT_RUNTIME_ITERATIONS,
3334
};
3435
use collector::toolchain::{
3536
create_toolchain_from_published_version, get_local_toolchain, Sysroot, Toolchain,
@@ -641,12 +642,16 @@ fn main_result() -> anyhow::Result<i32> {
641642
} else {
642643
CargoIsolationMode::Isolated
643644
};
644-
let runtime_suite = runtime::prepare_runtime_benchmark_suite(
645-
&toolchain,
645+
646+
let mut conn = rt.block_on(pool.connection());
647+
let artifact_id = ArtifactId::Tag(toolchain.id.clone());
648+
let runtime_suite = rt.block_on(load_runtime_benchmarks(
649+
conn.as_mut(),
646650
&runtime_benchmark_dir,
647651
isolation_mode,
648-
)?;
649-
let artifact_id = ArtifactId::Tag(toolchain.id.clone());
652+
&toolchain,
653+
&artifact_id,
654+
))?;
650655

651656
let shared = SharedBenchmarkConfig {
652657
artifact_id,
@@ -657,7 +662,6 @@ fn main_result() -> anyhow::Result<i32> {
657662
filter: BenchmarkFilter::new(local.exclude, local.include),
658663
iterations,
659664
};
660-
let conn = rt.block_on(pool.connection());
661665
run_benchmarks(&mut rt, conn, shared, None, Some(config))?;
662666
Ok(0)
663667
}
@@ -946,6 +950,35 @@ fn main_result() -> anyhow::Result<i32> {
946950
}
947951
}
948952

953+
async fn load_runtime_benchmarks(
954+
conn: &mut dyn Connection,
955+
benchmark_dir: &Path,
956+
isolation_mode: CargoIsolationMode,
957+
toolchain: &Toolchain,
958+
artifact_id: &ArtifactId,
959+
) -> anyhow::Result<BenchmarkSuite> {
960+
let BenchmarkSuiteCompilation {
961+
suite,
962+
failed_to_compile,
963+
} = runtime::prepare_runtime_benchmark_suite(toolchain, benchmark_dir, isolation_mode)?;
964+
965+
record_runtime_compilation_errors(conn, artifact_id, failed_to_compile).await;
966+
Ok(suite)
967+
}
968+
969+
async fn record_runtime_compilation_errors(
970+
connection: &mut dyn Connection,
971+
artifact_id: &ArtifactId,
972+
errors: HashMap<String, String>,
973+
) {
974+
let artifact_row_number = connection.artifact_id(artifact_id).await;
975+
for (krate, error) in errors {
976+
connection
977+
.record_error(artifact_row_number, &krate, &error)
978+
.await;
979+
}
980+
}
981+
949982
fn log_db(db_option: &DbOption) {
950983
println!("Using database `{}`", db_option.db);
951984
}
@@ -1025,7 +1058,7 @@ fn run_benchmarks(
10251058

10261059
/// Perform benchmarks on a published artifact.
10271060
fn bench_published_artifact(
1028-
connection: Box<dyn Connection>,
1061+
mut connection: Box<dyn Connection>,
10291062
rt: &mut Runtime,
10301063
toolchain: Toolchain,
10311064
dirs: &BenchmarkDirs,
@@ -1047,11 +1080,13 @@ fn bench_published_artifact(
10471080
let mut compile_benchmarks = get_compile_benchmarks(dirs.compile, None, None, None)?;
10481081
compile_benchmarks.retain(|b| b.category().is_stable());
10491082

1050-
let runtime_suite = runtime::prepare_runtime_benchmark_suite(
1051-
&toolchain,
1052-
dirs.runtime,
1083+
let runtime_suite = rt.block_on(load_runtime_benchmarks(
1084+
connection.as_mut(),
1085+
&dirs.runtime,
10531086
CargoIsolationMode::Isolated,
1054-
)?;
1087+
&toolchain,
1088+
&artifact_id,
1089+
))?;
10551090

10561091
let shared = SharedBenchmarkConfig {
10571092
artifact_id,

‎collector/src/lib.rs

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -268,8 +268,12 @@ impl CollectorStepBuilder {
268268
}
269269

270270
pub fn record_runtime_benchmarks(mut self, suite: &BenchmarkSuite) -> Self {
271-
self.steps
272-
.extend(suite.groups.iter().map(runtime_group_step_name));
271+
self.steps.extend(
272+
suite
273+
.groups
274+
.iter()
275+
.map(|group| runtime_group_step_name(&group.name)),
276+
);
273277
self
274278
}
275279

@@ -321,18 +325,18 @@ impl CollectorCtx {
321325
conn: &dyn Connection,
322326
group: &BenchmarkGroup,
323327
) -> Option<String> {
324-
let step_name = runtime_group_step_name(group);
328+
let step_name = runtime_group_step_name(&group.name);
325329
conn.collector_start_step(self.artifact_row_id, &step_name)
326330
.await
327331
.then_some(step_name)
328332
}
329333

330334
pub async fn end_runtime_step(&self, conn: &dyn Connection, group: &BenchmarkGroup) {
331-
conn.collector_end_step(self.artifact_row_id, &runtime_group_step_name(group))
335+
conn.collector_end_step(self.artifact_row_id, &runtime_group_step_name(&group.name))
332336
.await
333337
}
334338
}
335339

336-
fn runtime_group_step_name(group: &BenchmarkGroup) -> String {
337-
format!("runtime:{}", group.name)
340+
pub fn runtime_group_step_name(benchmark_name: &str) -> String {
341+
format!("runtime:{}", benchmark_name)
338342
}

‎collector/src/runtime/benchmark.rs

Lines changed: 48 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
use crate::runtime_group_step_name;
12
use crate::toolchain::Toolchain;
23
use anyhow::Context;
34
use benchlib::benchmark::passes_filter;
@@ -90,6 +91,12 @@ pub enum CargoIsolationMode {
9091
Isolated,
9192
}
9293

94+
pub struct BenchmarkSuiteCompilation {
95+
pub suite: BenchmarkSuite,
96+
// Maps benchmark group name to compilation error
97+
pub failed_to_compile: HashMap<String, String>,
98+
}
99+
93100
/// Find all runtime benchmark crates in `benchmark_dir` and compile them.
94101
/// We assume that each binary defines a benchmark suite using `benchlib`.
95102
/// We then execute each benchmark suite with the `list-benchmarks` command to find out its
@@ -98,7 +105,7 @@ pub fn prepare_runtime_benchmark_suite(
98105
toolchain: &Toolchain,
99106
benchmark_dir: &Path,
100107
isolation_mode: CargoIsolationMode,
101-
) -> anyhow::Result<BenchmarkSuite> {
108+
) -> anyhow::Result<BenchmarkSuiteCompilation> {
102109
let benchmark_crates = get_runtime_benchmark_groups(benchmark_dir)?;
103110

104111
let temp_dir: Option<TempDir> = match isolation_mode {
@@ -120,6 +127,7 @@ pub fn prepare_runtime_benchmark_suite(
120127
println!("Compiling {group_count} runtime benchmark groups");
121128

122129
let mut groups = Vec::new();
130+
let mut failed_to_compile = HashMap::new();
123131
for (index, benchmark_crate) in benchmark_crates.into_iter().enumerate() {
124132
println!(
125133
"Compiling {:<22} ({}/{group_count})",
@@ -129,25 +137,41 @@ pub fn prepare_runtime_benchmark_suite(
129137

130138
let target_dir = temp_dir.as_ref().map(|d| d.path());
131139

132-
let cargo_process = start_cargo_build(toolchain, &benchmark_crate.path, target_dir)
140+
let result = start_cargo_build(toolchain, &benchmark_crate.path, target_dir)
133141
.with_context(|| {
134142
anyhow::anyhow!("Cannot start compilation of {}", benchmark_crate.name)
135-
})?;
136-
let group =
137-
parse_benchmark_group(cargo_process, &benchmark_crate.name).with_context(|| {
138-
anyhow::anyhow!("Cannot compile runtime benchmark {}", benchmark_crate.name)
139-
})?;
140-
groups.push(group);
143+
})
144+
.and_then(|process| {
145+
parse_benchmark_group(process, &benchmark_crate.name).with_context(|| {
146+
anyhow::anyhow!("Cannot compile runtime benchmark {}", benchmark_crate.name)
147+
})
148+
});
149+
match result {
150+
Ok(group) => groups.push(group),
151+
Err(error) => {
152+
log::error!(
153+
"Cannot compile runtime benchmark group `{}`",
154+
benchmark_crate.name
155+
);
156+
failed_to_compile.insert(
157+
runtime_group_step_name(&benchmark_crate.name),
158+
format!("{error:?}"),
159+
);
160+
}
161+
}
141162
}
142163

143164
groups.sort_unstable_by(|a, b| a.binary.cmp(&b.binary));
144165
log::debug!("Found binaries: {:?}", groups);
145166

146167
check_duplicates(&groups)?;
147168

148-
Ok(BenchmarkSuite {
149-
groups,
150-
_tmp_artifacts_dir: temp_dir,
169+
Ok(BenchmarkSuiteCompilation {
170+
suite: BenchmarkSuite {
171+
groups,
172+
_tmp_artifacts_dir: temp_dir,
173+
},
174+
failed_to_compile,
151175
})
152176
}
153177

@@ -181,6 +205,7 @@ fn parse_benchmark_group(
181205
let mut group: Option<BenchmarkGroup> = None;
182206

183207
let stream = BufReader::new(cargo_process.stdout.take().unwrap());
208+
let mut messages = String::new();
184209
for message in Message::parse_stream(stream) {
185210
let message = message?;
186211
match message {
@@ -210,25 +235,28 @@ fn parse_benchmark_group(
210235
}
211236
}
212237
}
213-
Message::TextLine(line) => println!("{}", line),
238+
Message::TextLine(line) => {
239+
println!("{line}")
240+
}
214241
Message::CompilerMessage(msg) => {
215-
print!("{}", msg.message.rendered.unwrap_or(msg.message.message))
242+
let message = msg.message.rendered.unwrap_or(msg.message.message);
243+
messages.push_str(&message);
244+
print!("{message}");
216245
}
217246
_ => {}
218247
}
219248
}
220249

221-
let group = group.ok_or_else(|| {
222-
anyhow::anyhow!("Runtime benchmark group `{group_name}` has not produced any binary")
223-
})?;
224-
225250
let output = cargo_process.wait()?;
226251
if !output.success() {
227252
Err(anyhow::anyhow!(
228-
"Failed to compile runtime benchmark, exit code {}",
229-
output.code().unwrap_or(1)
253+
"Failed to compile runtime benchmark, exit code {}\n{messages}",
254+
output.code().unwrap_or(1),
230255
))
231256
} else {
257+
let group = group.ok_or_else(|| {
258+
anyhow::anyhow!("Runtime benchmark group `{group_name}` has not produced any binary")
259+
})?;
232260
Ok(group)
233261
}
234262
}
@@ -246,7 +274,7 @@ fn start_cargo_build(
246274
.arg("build")
247275
.arg("--release")
248276
.arg("--message-format")
249-
.arg("json-diagnostic-rendered-ansi")
277+
.arg("json-diagnostic-short")
250278
.current_dir(benchmark_dir)
251279
.stdin(Stdio::null())
252280
.stdout(Stdio::piped())

‎collector/src/runtime/mod.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ use thousands::Separable;
88
use benchlib::comm::messages::{BenchmarkMessage, BenchmarkResult, BenchmarkStats};
99
pub use benchmark::{
1010
prepare_runtime_benchmark_suite, runtime_benchmark_dir, BenchmarkFilter, BenchmarkGroup,
11-
BenchmarkSuite, CargoIsolationMode,
11+
BenchmarkSuite, BenchmarkSuiteCompilation, CargoIsolationMode,
1212
};
1313
use database::{ArtifactIdNumber, CollectionId, Connection};
1414

0 commit comments

Comments
 (0)