Create a new endpoint for compilation sections

A separate endpoint was created, because loading the sections requires loading self-profile data, which can be quite slow.
Making it a separate endpoint won't block loading of the graphs, which are loaded quite fast.

Author: Kobzol

site/frontend/src/urls.ts

@@ -7,5 +7,5 @@ export const STATUS_DATA_URL = `${BASE_URL}/status_page`;
 export const BOOTSTRAP_DATA_URL = `${BASE_URL}/bootstrap`;
 export const GRAPH_DATA_URL = `${BASE_URL}/graphs`;
 export const COMPARE_DATA_URL = `${BASE_URL}/get`;
-export const COMPARE_COMPILE_DETAIL_DATA_URL = `${BASE_URL}/compare-compile-detail`;
+export const COMPARE_COMPILE_DETAIL_DATA_URL = `${BASE_URL}/compare-compile-detail-graphs`;
 export const SELF_PROFILE_DATA_URL = `${BASE_URL}/self-profile`;

site/src/api.rs

@@ -110,7 +110,7 @@ pub mod graphs {
     }
 }
 
-pub mod detail {
+pub mod detail_graphs {
     use crate::api::graphs::{GraphKind, Series};
     use collector::Bound;
     use serde::de::{DeserializeOwned, Error};
@@ -159,13 +159,48 @@ pub mod detail {
         deserializer.deserialize_str(CommaSeparatedVisitor(Default::default()))
     }
 
-    #[derive(Debug, PartialEq, Clone, Serialize)]
+    #[derive(Debug, Serialize)]
     pub struct Response {
         pub commits: Vec<(i64, String)>,
         pub graphs: Vec<Series>,
     }
 }
 
+pub mod detail_sections {
+    use collector::Bound;
+    use serde::{Deserialize, Serialize};
+
+    #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
+    pub struct Request {
+        pub start: Bound,
+        pub end: Bound,
+        pub benchmark: String,
+        pub scenario: String,
+        pub profile: String,
+    }
+
+    #[derive(Default, Debug, Clone, Serialize)]
+    pub struct CompilationSection {
+        pub name: String,
+        // It is unspecified if this is duration, fraction or something else. It should only be
+        // evaluated against the total sum of values.
+        pub value: u64,
+    }
+
+    /// Counts how much <resource> (time/instructions) was spent in individual compilation sections
+    /// (e.g. frontend, backend, linking) during the compilation of a single test case.
+    #[derive(Default, Debug, Serialize)]
+    pub struct CompilationSections {
+        pub sections: Vec<CompilationSection>,
+    }
+
+    #[derive(Debug, Serialize)]
+    pub struct Response {
+        pub before: Option<CompilationSections>,
+        pub after: Option<CompilationSections>,
+    }
+}
+
 pub mod bootstrap {
     use collector::Bound;
     use hashbrown::HashMap;

site/src/request_handlers.rs

@@ -9,7 +9,7 @@ mod status_page;
 pub use bootstrap::handle_bootstrap;
 pub use dashboard::handle_dashboard;
 pub use github::handle_github;
-pub use graph::{handle_compile_detail, handle_graphs};
+pub use graph::{handle_compile_detail_graphs, handle_compile_detail_sections, handle_graphs};
 pub use next_artifact::handle_next_artifact;
 pub use self_profile::{
     handle_self_profile, handle_self_profile_processed_download, handle_self_profile_raw,

site/src/request_handlers/graph.rs

@@ -1,34 +1,38 @@
-use collector::Bound;
 use std::collections::HashMap;
 use std::sync::Arc;
 
+use collector::Bound;
+
+use crate::api::detail_sections::CompilationSections;
 use crate::api::graphs::GraphKind;
-use crate::api::{detail, graphs, ServerResult};
+use crate::api::{detail_graphs, detail_sections, graphs, ServerResult};
 use crate::db::{self, ArtifactId, Profile, Scenario};
 use crate::interpolate::IsInterpolated;
 use crate::load::SiteCtxt;
 use crate::selector::{CompileBenchmarkQuery, CompileTestCase, Selector, SeriesResponse};
+use crate::self_profile::get_or_download_self_profile;
 
-/// Returns data for a detailed information when comparing a single test result comparison
+/// Returns data for before/after graphs when comparing a single test result comparison
 /// for a compile-time benchmark.
-pub async fn handle_compile_detail(
-    request: detail::Request,
+pub async fn handle_compile_detail_graphs(
+    request: detail_graphs::Request,
     ctxt: Arc<SiteCtxt>,
-) -> ServerResult<detail::Response> {
-    log::info!("handle_compile_detail({:?})", request);
+) -> ServerResult<detail_graphs::Response> {
+    log::info!("handle_compile_detail_graphs({:?})", request);
 
     let artifact_ids = Arc::new(master_artifact_ids_for_range(
         &ctxt,
         request.start,
         request.end,
     ));
 
+    let scenario = request.scenario.parse()?;
     let interpolated_responses: Vec<_> = ctxt
         .statistic_series(
             CompileBenchmarkQuery::default()
-                .benchmark(Selector::One(request.benchmark))
+                .benchmark(Selector::One(request.benchmark.clone()))
                 .profile(Selector::One(request.profile.parse()?))
-                .scenario(Selector::One(request.scenario.parse()?))
+                .scenario(Selector::One(scenario))
                 .metric(Selector::One(request.stat.parse()?)),
             artifact_ids.clone(),
         )
@@ -48,12 +52,73 @@ pub async fn handle_compile_detail(
     }
     assert!(interpolated_responses.next().is_none());
 
-    Ok(detail::Response {
+    Ok(detail_graphs::Response {
         commits: artifact_ids_to_commits(artifact_ids),
         graphs,
     })
 }
 
+/// Returns data for compilation sections (frontend/backend/linker) when comparing a single test
+/// result comparison for a compile-time benchmark.
+pub async fn handle_compile_detail_sections(
+    request: detail_sections::Request,
+    ctxt: Arc<SiteCtxt>,
+) -> ServerResult<detail_sections::Response> {
+    log::info!("handle_compile_detail_sections({:?})", request);
+
+    let artifact_ids = Arc::new(master_artifact_ids_for_range(
+        &ctxt,
+        request.start,
+        request.end,
+    ));
+
+    let scenario = request.scenario.parse()?;
+
+    async fn calculate_sections(
+        ctxt: &SiteCtxt,
+        aid: Option<&ArtifactId>,
+        benchmark: &str,
+        profile: &str,
+        scenario: Scenario,
+    ) -> Option<CompilationSections> {
+        match aid {
+            Some(aid) => {
+                get_or_download_self_profile(ctxt, aid.clone(), benchmark, profile, scenario, None)
+                    .await
+                    .ok()
+                    .map(|profile| CompilationSections {
+                        sections: profile.compilation_sections,
+                    })
+            }
+            None => None,
+        }
+    }
+
+    // Doc queries are not split into the classic frontend/backend/linker parts.
+    let (before, after) = if request.profile != "doc" {
+        tokio::join!(
+            calculate_sections(
+                &ctxt,
+                artifact_ids.get(0),
+                &request.benchmark,
+                &request.profile,
+                scenario,
+            ),
+            calculate_sections(
+                &ctxt,
+                artifact_ids.get(1),
+                &request.benchmark,
+                &request.profile,
+                scenario,
+            )
+        )
+    } else {
+        (None, None)
+    };
+
+    Ok(detail_sections::Response { before, after })
+}
+
 pub async fn handle_graphs(
     request: graphs::Request,
     ctxt: Arc<SiteCtxt>,

site/src/request_handlers/self_profile.rs

@@ -14,10 +14,7 @@ use crate::comparison::Metric;
 use crate::db::ArtifactId;
 use crate::load::SiteCtxt;
 use crate::selector::{self};
-use crate::self_profile::{
-    download_and_analyze_self_profile, get_self_profile_raw_data, SelfProfileKey,
-    SelfProfileWithAnalysis,
-};
+use crate::self_profile::{get_or_download_self_profile, get_self_profile_raw_data};
 use crate::server::{Response, ResponseHeaders};
 
 pub async fn handle_self_profile_processed_download(
@@ -535,35 +532,7 @@ pub async fn handle_self_profile(
     assert_eq!(cpu_responses.len(), 1, "all selectors are exact");
     let mut cpu_response = cpu_responses.remove(0).series;
 
-    async fn get_from_cache(
-        ctxt: &SiteCtxt,
-        aid: ArtifactId,
-        benchmark: &str,
-        profile: &str,
-        scenario: database::Scenario,
-        metric: Option<f64>,
-    ) -> ServerResult<SelfProfileWithAnalysis> {
-        let key = SelfProfileKey {
-            aid: aid.clone(),
-            benchmark: benchmark.to_string(),
-            profile: profile.to_string(),
-            scenario,
-        };
-        let cache_result = ctxt.self_profile_cache.lock().get(&key);
-        match cache_result {
-            Some(res) => Ok(res),
-            None => {
-                let profile = download_and_analyze_self_profile(
-                    ctxt, aid, benchmark, profile, scenario, metric,
-                )
-                .await?;
-                ctxt.self_profile_cache.lock().insert(key, profile.clone());
-                Ok(profile)
-            }
-        }
-    }
-
-    let mut self_profile = get_from_cache(
+    let mut self_profile = get_or_download_self_profile(
         ctxt,
         commits.get(0).unwrap().clone(),
         bench_name,
@@ -574,7 +543,7 @@ pub async fn handle_self_profile(
     .await?;
     let base_self_profile = match commits.get(1) {
         Some(aid) => Some(
-            get_from_cache(
+            get_or_download_self_profile(
                 ctxt,
                 aid.clone(),
                 bench_name,

site/src/self_profile.rs

@@ -1,9 +1,11 @@
 //! This module handles self-profile "rich" APIs (e.g., chrome profiler JSON)
 //! generation from the raw artifacts on demand.
 
+use crate::api::detail_sections::CompilationSection;
 use crate::api::self_profile::ArtifactSize;
 use crate::api::{self_profile, ServerResult};
 use crate::load::SiteCtxt;
+use analyzeme::ProfilingData;
 use anyhow::Context;
 use bytes::Buf;
 use database::ArtifactId;
@@ -197,9 +199,10 @@ impl SelfProfileCache {
 pub struct SelfProfileWithAnalysis {
     pub profile: self_profile::SelfProfile,
     pub profiling_data: analyzeme::AnalysisResults,
+    pub compilation_sections: Vec<CompilationSection>,
 }
 
-pub(crate) async fn download_and_analyze_self_profile(
+async fn download_and_analyze_self_profile(
     ctxt: &SiteCtxt,
     aid: ArtifactId,
     benchmark: &str,
@@ -221,15 +224,101 @@ pub(crate) async fn download_and_analyze_self_profile(
                 .map_err(|e| format!("error extracting self profiling data: {}", e))?,
             Err(e) => return Err(format!("could not fetch raw profile data: {e:?}")),
         };
+
+    let compilation_sections = compute_compilation_sections(&profiling_data);
     let profiling_data = profiling_data.perform_analysis();
     let profile =
         get_self_profile_data(metric, &profiling_data).map_err(|e| format!("{}: {}", aid, e))?;
     Ok(SelfProfileWithAnalysis {
         profile,
         profiling_data,
+        compilation_sections,
     })
 }
 
+/// Tries to categorize the duration of three high-level sections of compilation (frontend,
+/// backend, linker) from the self-profile queries.
+fn compute_compilation_sections(profile: &ProfilingData) -> Vec<CompilationSection> {
+    let mut first_event_start = None;
+    let mut frontend_end = None;
+    let mut backend_start = None;
+    let mut backend_end = None;
+    let mut linker_duration = None;
+
+    for event in profile.iter_full() {
+        if first_event_start.is_none() {
+            first_event_start = event.payload.timestamp().map(|t| t.start());
+        }
+
+        if event.label == "analysis" {
+            // End of "analysis" => end of frontend
+            frontend_end = event.payload.timestamp().map(|t| t.end());
+        } else if event.label == "codegen_crate" {
+            // Start of "codegen_crate" => start of backend
+            backend_start = event.payload.timestamp().map(|t| t.start());
+        } else if event.label == "finish_ongoing_codegen" {
+            // End of "finish_ongoing_codegen" => end of backend
+            backend_end = event.payload.timestamp().map(|t| t.end());
+        } else if event.label == "link_crate" {
+            // The "link" query overlaps codegen, so we want to look at the "link_crate" query
+            // instead.
+            linker_duration = event.duration();
+        }
+    }
+    let mut sections = vec![];
+    if let (Some(start), Some(end)) = (first_event_start, frontend_end) {
+        if let Ok(duration) = end.duration_since(start) {
+            sections.push(CompilationSection {
+                name: "Frontend".to_string(),
+                value: duration.as_nanos() as u64,
+            });
+        }
+    }
+    if let (Some(start), Some(end)) = (backend_start, backend_end) {
+        if let Ok(duration) = end.duration_since(start) {
+            sections.push(CompilationSection {
+                name: "Backend".to_string(),
+                value: duration.as_nanos() as u64,
+            });
+        }
+    }
+    if let Some(duration) = linker_duration {
+        sections.push(CompilationSection {
+            name: "Linker".to_string(),
+            value: duration.as_nanos() as u64,
+        });
+    }
+
+    sections
+}
+
+pub(crate) async fn get_or_download_self_profile(
+    ctxt: &SiteCtxt,
+    aid: ArtifactId,
+    benchmark: &str,
+    profile: &str,
+    scenario: database::Scenario,
+    metric: Option<f64>,
+) -> ServerResult<SelfProfileWithAnalysis> {
+    let key = SelfProfileKey {
+        aid: aid.clone(),
+        benchmark: benchmark.to_string(),
+        profile: profile.to_string(),
+        scenario,
+    };
+    let cache_result = ctxt.self_profile_cache.lock().get(&key);
+    match cache_result {
+        Some(res) => Ok(res),
+        None => {
+            let profile =
+                download_and_analyze_self_profile(ctxt, aid, benchmark, profile, scenario, metric)
+                    .await?;
+            ctxt.self_profile_cache.lock().insert(key, profile.clone());
+            Ok(profile)
+        }
+    }
+}
+
 fn get_self_profile_data(
     cpu_clock: Option<f64>,
     profile: &analyzeme::AnalysisResults,