gix-corpus now respects the --trace flag

Author: Byron

Cargo.lock

@@ -23,7 +23,7 @@ estimate-hours = ["dep:itertools", "dep:fs-err", "dep:crossbeam-channel", "dep:s
 query = ["dep:rusqlite"]
 ## Run algorithms on a corpus of repositories and store their results for later comparison and intelligence gathering.
 ## *Note that* `organize` we need for finding git repositories fast.
-corpus = [ "dep:rusqlite", "dep:sysinfo", "organize", "dep:crossbeam-channel", "dep:serde_json", "dep:tracing-forest", "dep:tracing-subscriber", "dep:tracing" ]
+corpus = [ "dep:rusqlite", "dep:sysinfo", "organize", "dep:crossbeam-channel", "dep:serde_json", "dep:tracing-forest", "dep:tracing-subscriber", "dep:tracing", "dep:parking_lot" ]
 
 #! ### Mutually Exclusive Networking
 #! If both are set, _blocking-client_ will take precedence, allowing `--all-features` to be used.
@@ -72,6 +72,7 @@ smallvec = { version = "1.10.0", optional = true }
 rusqlite = { version = "0.29.0", optional = true, features = ["bundled"] }
 
 # for 'corpus'
+parking_lot = { version = "0.12.1", optional = true }
 sysinfo = { version = "0.29.2", optional = true, default-features = false }
 serde_json = { version = "1.0.65", optional = true }
 tracing-forest = { version = "0.1.5", features = ["serde"], optional = true }

gitoxide-core/Cargo.toml

@@ -12,12 +12,20 @@ use std::time::{Duration, Instant};
 
 impl Engine {
     /// Open the corpus DB or create it.
-    pub fn open_or_create(db: PathBuf, gitoxide_version: String, progress: corpus::Progress) -> anyhow::Result<Engine> {
+    pub fn open_or_create(
+        db: PathBuf,
+        gitoxide_version: String,
+        progress: corpus::Progress,
+        trace_to_progress: bool,
+        reverse_trace_lines: bool,
+    ) -> anyhow::Result<Engine> {
         let con = crate::corpus::db::create(db).context("Could not open or create database")?;
         Ok(Engine {
             progress,
             con,
             gitoxide_version,
+            trace_to_progress,
+            reverse_trace_lines,
         })
     }
 
@@ -66,7 +74,11 @@ impl Engine {
 
             if task.execute_exclusive || threads == 1 {
                 let mut run_progress = repo_progress.add_child("set later");
-                let (_guard, current_id) = corpus::trace::override_thread_subscriber(db_path.as_str())?;
+                let (_guard, current_id) = corpus::trace::override_thread_subscriber(
+                    db_path.as_str(),
+                    self.trace_to_progress.then(|| task_progress.add_child("trace")),
+                    self.reverse_trace_lines,
+                )?;
 
                 for repo in &repos {
                     if gix::interrupt::is_triggered() {
@@ -80,7 +92,7 @@ impl Engine {
                             .display()
                     ));
 
-                    // TODO: wait for new release to be able to provide run_id via span attributes
+                    // TODO: wait for new release of `tracing-forest` to be able to provide run_id via span attributes
                     let mut run = Self::insert_run(&self.con, gitoxide_id, runner_id, *task_id, repo.id)?;
                     current_id.store(run.id, Ordering::SeqCst);
                     tracing::info_span!("run", run_id = run.id).in_scope(|| {
@@ -106,9 +118,11 @@ impl Engine {
                         let shared_repo_progress = repo_progress.clone();
                         let db_path = db_path.clone();
                         move |tid| {
+                            let mut progress = gix::threading::lock(&shared_repo_progress);
                             (
-                                corpus::trace::override_thread_subscriber(db_path.as_str()),
-                                gix::threading::lock(&shared_repo_progress).add_child(format!("{tid}")),
+                                // threaded printing is usually spammy, and lines interleave so it's useless.
+                                corpus::trace::override_thread_subscriber(db_path.as_str(), None, false),
+                                progress.add_child(format!("{tid}")),
                                 rusqlite::Connection::open(&db_path),
                             )
                         }

gitoxide-core/src/corpus/engine.rs

@@ -5,6 +5,8 @@ pub struct Engine {
     progress: Progress,
     con: rusqlite::Connection,
     gitoxide_version: String,
+    trace_to_progress: bool,
+    reverse_trace_lines: bool,
 }
 
 pub struct RunOutcome {

gitoxide-core/src/corpus/mod.rs

@@ -1,37 +1,64 @@
+use gix::progress::DoOrDiscard;
+use parking_lot::Mutex;
 use rusqlite::params;
 use std::path::Path;
 use std::sync::atomic::{AtomicU32, Ordering};
-use std::sync::{Arc, Mutex};
+use std::sync::Arc;
 use tracing_forest::tree::Tree;
 use tracing_subscriber::layer::SubscriberExt;
 
+type ProgressItem = DoOrDiscard<gix::progress::prodash::tree::Item>;
+
 pub fn override_thread_subscriber(
     db_path: impl AsRef<Path>,
+    progress: Option<ProgressItem>,
+    reverse_lines: bool,
 ) -> anyhow::Result<(tracing::subscriber::DefaultGuard, Arc<AtomicU32>)> {
     let current_id = Arc::new(AtomicU32::default());
     let processor = tracing_forest::Printer::new().formatter(StoreTreeToDb {
         con: Arc::new(Mutex::new(rusqlite::Connection::open(&db_path)?)),
         run_id: current_id.clone(),
+        progress: progress.map(Mutex::new),
+        reverse_lines,
     });
     let subscriber = tracing_subscriber::Registry::default().with(tracing_forest::ForestLayer::from(processor));
     let guard = tracing::subscriber::set_default(subscriber);
     Ok((guard, current_id))
 }
 
 pub struct StoreTreeToDb {
-    pub con: Arc<Mutex<rusqlite::Connection>>,
-    pub run_id: Arc<AtomicU32>,
+    con: Arc<Mutex<rusqlite::Connection>>,
+    run_id: Arc<AtomicU32>,
+    progress: Option<Mutex<ProgressItem>>,
+    reverse_lines: bool,
 }
 
 impl tracing_forest::printer::Formatter for StoreTreeToDb {
     type Error = rusqlite::Error;
 
     fn fmt(&self, tree: &Tree) -> Result<String, Self::Error> {
+        if let Some((progress, tree)) = self
+            .progress
+            .as_ref()
+            .map(Mutex::lock)
+            .zip(tracing_forest::printer::Pretty.fmt(tree).ok())
+        {
+            use gix::Progress;
+            if self.reverse_lines {
+                for line in tree.lines().rev() {
+                    progress.info(line);
+                }
+            } else {
+                for line in tree.lines() {
+                    progress.info(line);
+                }
+            }
+        }
+        // TODO: wait for new release of `tracing-forest` and load the ID from span fields.
         let json = serde_json::to_string_pretty(&tree).expect("serialization to string always works");
         let run_id = self.run_id.load(Ordering::SeqCst);
         self.con
             .lock()
-            .unwrap()
             .execute("UPDATE run SET spans_json = ?1 WHERE id = ?2", params![json, run_id])?;
         Ok(String::new())
     }

gitoxide-core/src/corpus/trace.rs

@@ -136,21 +136,30 @@ pub fn main() -> Result<()> {
 
     match cmd {
         #[cfg(feature = "gitoxide-core-tools-corpus")]
-        Subcommands::Corpus(crate::plumbing::options::corpus::Platform { db, path, cmd }) => prepare_and_run(
-            "corpus",
-            trace,
-            auto_verbose,
-            progress,
-            progress_keep_open,
-            core::corpus::PROGRESS_RANGE,
-            move |progress, _out, _err| {
-                let mut engine = core::corpus::Engine::open_or_create(db, env!("GITOXIDE_VERSION").into(), progress)?;
-                match cmd {
-                    crate::plumbing::options::corpus::SubCommands::Run => engine.run(path, thread_limit),
-                    crate::plumbing::options::corpus::SubCommands::Refresh => engine.refresh(path),
-                }
-            },
-        ),
+        Subcommands::Corpus(crate::plumbing::options::corpus::Platform { db, path, cmd }) => {
+            let reverse_trace_lines = progress;
+            prepare_and_run(
+                "corpus",
+                trace,
+                auto_verbose,
+                progress,
+                progress_keep_open,
+                core::corpus::PROGRESS_RANGE,
+                move |progress, _out, _err| {
+                    let mut engine = core::corpus::Engine::open_or_create(
+                        db,
+                        env!("GITOXIDE_VERSION").into(),
+                        progress,
+                        trace,
+                        reverse_trace_lines,
+                    )?;
+                    match cmd {
+                        crate::plumbing::options::corpus::SubCommands::Run => engine.run(path, thread_limit),
+                        crate::plumbing::options::corpus::SubCommands::Refresh => engine.refresh(path),
+                    }
+                },
+            )
+        }
         Subcommands::CommitGraph(cmd) => match cmd {
             commitgraph::Subcommands::List { spec } => prepare_and_run(
                 "commitgraph-list",

src/plumbing/main.rs

@@ -108,8 +108,8 @@ pub mod pretty {
                 move |tree: &tracing_forest::tree::Tree| -> Result<String, std::fmt::Error> {
                     use gix::Progress;
                     use tracing_forest::Formatter;
-                    let tree = tracing_forest::printer::Pretty.fmt(tree)?;
                     let progress = &mut progress.lock().unwrap();
+                    let tree = tracing_forest::printer::Pretty.fmt(tree)?;
                     if reverse_lines {
                         for line in tree.lines().rev() {
                             progress.info(line);