feat: support workspace filters when fetching them from the object database

Author: Byron

Cargo.lock

@@ -293,16 +293,27 @@ The top-level crate that acts as hub to all functionality provided by the `gix-*
 Check out the [performance discussion][gix-diff-performance] as well.
 
 * **tree**
-  * [x] changes needed to obtain _other tree_
+    * [x] changes needed to obtain _other tree_
 * **patches**    
-  * There are various ways to generate a patch from two blobs.
-  * [ ] any
+    * There are various ways to generate a patch from two blobs.
+    * [ ] text
+    * [ ] binary
 * **lines**
-  * [x] Simple line-by-line diffs powered by the `imara-diff` crate.
-* diffing, merging, working with hunks of data
-* find differences between various states, i.e. index, working tree, commit-tree
+    * [x] Simple line-by-line diffs powered by the `imara-diff` crate.
+* **generic rename tracker to find renames and copies**
+    * [x] find by exact match
+    * [x] find by similarity check
+    * [ ] heuristics to find best candidate
+    * [ ] find by basename to help detecting simple moves
+* **blob**
+    * [ ] worktree conversions
+    * [ ] `textconv` filters
+    * [ ] caching of diff-able data
+    * [ ] special handling of files beyond the big-file threshold.
+    * [ ] detection of binary files by looking at header (first 8k bytes)
+* [ ] working with hunks of data
 * [x] API documentation
-  * [ ] Examples
+    * [ ] Examples
 
 [gix-diff-performance]: https://github.com/Byron/gitoxide/discussions/74
 

crate-status.md

@@ -13,7 +13,7 @@ autotests = false
 [features]
 default = ["blob"]
 ## Enable diffing of blobs using imara-diff, which also allows for a generic rewrite tracking implementation.
-blob = ["dep:imara-diff"]
+blob = ["dep:imara-diff", "dep:gix-filter", "dep:gix-worktree"]
 ## Data structures implement `serde::Serialize` and `serde::Deserialize`.
 serde = ["dep:serde", "gix-hash/serde", "gix-object/serde"]
 ## Make it possible to compile to the `wasm32-unknown-unknown` target.
@@ -25,6 +25,8 @@ doctest = false
 [dependencies]
 gix-hash = { version = "^0.13.1", path = "../gix-hash" }
 gix-object = { version = "^0.38.0", path = "../gix-object" }
+gix-filter = { version = "^0.6.0", path = "../gix-filter", optional = true }
+gix-worktree = { version = "^0.27.0", path = "../gix-worktree", default-features = false, features = ["attributes"], optional = true }
 
 thiserror = "1.0.32"
 imara-diff = { version = "0.1.3", optional = true }

gix-diff/Cargo.toml

@@ -0,0 +1,142 @@
+//! For using text diffs, please have a look at the [`imara-diff` documentation](https://docs.rs/imara-diff),
+//! maintained by [Pascal Kuthe](https://github.com/pascalkuthe).
+pub use imara_diff::*;
+use std::collections::HashMap;
+use std::path::PathBuf;
+
+use bstr::BString;
+
+/// Information about the diff performed to detect similarity.
+#[derive(Debug, Default, Clone, Copy, PartialEq, PartialOrd)]
+pub struct DiffLineStats {
+    /// The amount of lines to remove from the source to get to the destination.
+    pub removals: u32,
+    /// The amount of lines to add to the source to get to the destination.
+    pub insertions: u32,
+    /// The amount of lines of the previous state, in the source.
+    pub before: u32,
+    /// The amount of lines of the new state, in the destination.
+    pub after: u32,
+    /// A range from 0 to 1.0, where 1.0 is a perfect match and 0.5 is a similarity of 50%.
+    /// Similarity is the ratio between all lines in the previous blob and the current blob,
+    /// calculated as `(old_lines_count - new_lines_count) as f32 / old_lines_count.max(new_lines_count) as f32`.
+    pub similarity: f32,
+}
+
+/// A way to classify a resource suitable for diffing.
+#[derive(Copy, Clone, Debug, Ord, PartialOrd, Eq, PartialEq, Hash)]
+pub enum ResourceKind {
+    /// The source of a rewrite, rename or copy operation, or generally the old version of a resource.
+    OldOrSource,
+    /// The destination of a rewrite, rename or copy operation, or generally the new version of a resource.
+    NewOrDestination,
+}
+
+/// A set of values to define how to diff something that is associated with it using `git-attributes`, relevant for regular files.
+///
+/// Some values are related to diffing, some are related to conversions.
+#[derive(Debug, Clone)]
+pub struct Driver {
+    /// The name of the driver, as referred to by `[diff "name"]` in the git configuration.
+    pub name: BString,
+    /// The per-driver algorithm to use.
+    pub algorithm: Option<Algorithm>,
+    /// The external filter program to call like `<binary_to_text_command> /path/to/blob` which outputs a textual version of the provided
+    /// binary file.
+    /// Note that it's invoked with a shell if arguments are given.
+    pub binary_to_text_command: Option<BString>,
+    /// `true` if this driver deals with binary files, which means that a `binary_to_text_command` should be used to convert binary
+    /// into a textual representation.
+    pub is_binary: bool,
+}
+
+/// A way to access roots for different kinds of resources that are possibly located and accessible in a worktree.
+#[derive(Clone, Debug, Default)]
+pub struct WorktreeRoots {
+    /// A place where the source of a rewrite, rename or copy, or generally the previous version of resources, are located.
+    pub old_root: Option<PathBuf>,
+    /// A place where the destination of a rewrite, rename or copy, or generally the new version of resources, are located.
+    pub new_root: Option<PathBuf>,
+}
+
+/// A conversion pipeline to take an object or path from what's stored in `git` to what can be diffed, while
+/// following the guidance of git-attributes at the respective path to learn if diffing should happen or if
+/// the content is considered binary.
+///
+/// There are two different conversion flows, where the target of the flow is a buffer with diffable content:
+///
+/// * `worktree on disk` -> `text conversion`
+/// * `object` -> `worktree-filters` -> `text conversion`
+///
+/// Based on whether or not [`WorktreeRoots`] has the file in question, we either read directly from disk
+/// or transform from the object database.
+pub struct Pipeline {
+    /// A way to read data directly from the worktree.
+    pub roots: WorktreeRoots,
+    /// A pipeline to convert objects from what's stored in `git` to its worktree version.
+    pub worktree_filter: gix_filter::Pipeline,
+    /// Drivers to help customize the conversion behaviour depending on the location of items.
+    pub drivers: Vec<Driver>,
+    /// The amount of bytes that an object has to reach before being treated as binary.
+    /// These objects will not be queried, nor will their data be processed in any way.
+    pub large_file_threshold_bytes: u64,
+
+    /// Pre-configured attributes to obtain additional diff-related information.
+    attrs: gix_filter::attributes::search::Outcome,
+}
+
+/// A utility for performing a diff of two blobs, including flexible conversions, conversion-caching
+/// acquisition of diff information.
+/// Note that this instance will not call external filters as their output can't be known programmatically,
+/// but it allows to prepare their input if the caller wishes to perform this task.
+///
+/// Optimized for NxM lookups with built-in caching.
+pub struct Platform {
+    /// The old version of a diff-able blob, if set.
+    old: Option<platform::Diffable>,
+    /// The new version of a diff-able blob, if set.
+    new: Option<platform::Diffable>,
+
+    /// Options to alter how diffs should be performed.
+    pub options: platform::Options,
+    /// A way to convert objects into a diff-able format.
+    pub filter: Pipeline,
+    /// A way to access .gitattributes
+    pub attr_stack: gix_worktree::Stack,
+    /// A continuously growing cache keeping ready-for-diff blobs by their path in the worktree,
+    /// as that is what affects their final diff-able state.
+    ///
+    /// That way, expensive rewrite-checks with NxM matrix checks would be as fast as possible,
+    /// avoiding duplicate work.
+    diff_cache: HashMap<platform::CacheKey, platform::CacheValue>,
+}
+
+mod impls {
+    use crate::blob::{ResourceKind, WorktreeRoots};
+    use std::path::Path;
+
+    impl std::fmt::Display for ResourceKind {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            f.write_str(match self {
+                ResourceKind::OldOrSource => "old",
+                ResourceKind::NewOrDestination => "new",
+            })
+        }
+    }
+
+    impl WorktreeRoots {
+        /// Return the root path for the given `kind`
+        pub fn by_kind(&self, kind: ResourceKind) -> Option<&Path> {
+            match kind {
+                ResourceKind::OldOrSource => self.old_root.as_deref(),
+                ResourceKind::NewOrDestination => self.new_root.as_deref(),
+            }
+        }
+    }
+}
+
+///
+pub mod pipeline;
+
+///
+pub mod platform;

gix-diff/src/blob.rs

@@ -0,0 +1,79 @@
+use crate::blob::{Pipeline, ResourceKind, WorktreeRoots};
+use bstr::BStr;
+
+/// Data as part of an [Outcome].
+#[derive(Copy, Clone)]
+pub enum Data {
+    /// The data to use for diffing was written into the buffer that was passed during the call to [`Pipeline::convert_to_diffable()`].
+    Buffer,
+    /// The size that the binary blob had at the given revision, without having applied filters, as it's either
+    /// considered binary or above the big-file threshold.
+    ///
+    /// In this state, the binary file cannot be diffed.
+    Binary {
+        /// The size of the object prior to performing any filtering or as it was found on disk.
+        size: u64,
+    },
+}
+
+/// The outcome returned by [Pipeline::convert_to_diffable()](super::Pipeline::convert_to_diffable()).
+#[derive(Copy, Clone)]
+pub struct Outcome {
+    /// If available, an index into the `drivers` field to access more diff-related information of the driver for items
+    /// at the given path, as previously determined by git-attributes.
+    pub driver_index: Option<usize>,
+    /// The data itself, suitable for diffing.
+    pub data: Data,
+}
+
+///
+pub mod convert_to_diffable {
+    /// The error returned by [Pipeline::convert_to_diffable()](super::Pipeline::convert_to_diffable()).
+    #[derive(Debug, thiserror::Error)]
+    #[allow(missing_docs)]
+    pub enum Error {}
+}
+
+/// Lifecycle
+impl Pipeline {
+    /// Create a new instance of a pipeline which produces blobs suitable for diffing. `roots` allow to read worktree files directly, otherwise
+    /// `worktree_filter` is used to transform object database data directly. `drivers` further configure individual paths.
+    /// `large_file_threshold_bytes` is used to determine which objects or files are too large to process, or allow any size if this
+    /// is set to `0`.
+    pub fn new(
+        roots: WorktreeRoots,
+        worktree_filter: gix_filter::Pipeline,
+        drivers: Vec<super::Driver>,
+        large_file_threshold_bytes: u64,
+    ) -> Self {
+        Pipeline {
+            roots,
+            worktree_filter,
+            drivers,
+            large_file_threshold_bytes,
+            attrs: {
+                let mut out = gix_filter::attributes::search::Outcome::default();
+                out.initialize_with_selection(&Default::default(), Some("diff"));
+                out
+            },
+        }
+    }
+}
+
+/// Conversion
+impl Pipeline {
+    /// Convert the object at `id`, `mode`, `rela_path` and `kind`, providing access to `attributes` and `objects`.
+    /// The resulting diff-able data is written into `out`
+    pub fn convert_to_diffable(
+        &mut self,
+        id: &gix_hash::oid,
+        mode: gix_object::tree::EntryKind,
+        rela_path: &BStr,
+        kind: ResourceKind,
+        attributes: &mut dyn FnMut(&BStr, &mut gix_filter::attributes::search::Outcome),
+        objects: &dyn gix_object::FindObjectOrHeader,
+        out: &mut Vec<u8>,
+    ) -> Result<Outcome, convert_to_diffable::Error> {
+        todo!()
+    }
+}