feat: support workspace filters when fetching them from the object database

Author: Byron

Cargo.lock

@@ -293,16 +293,26 @@ The top-level crate that acts as hub to all functionality provided by the `gix-*
 Check out the [performance discussion][gix-diff-performance] as well.
 
 * **tree**
-  * [x] changes needed to obtain _other tree_
+    * [x] changes needed to obtain _other tree_
 * **patches**    
-  * There are various ways to generate a patch from two blobs.
-  * [ ] any
+    * There are various ways to generate a patch from two blobs.
+    * [ ] text
+    * [ ] binary
 * **lines**
-  * [x] Simple line-by-line diffs powered by the `imara-diff` crate.
-* diffing, merging, working with hunks of data
-* find differences between various states, i.e. index, working tree, commit-tree
+    * [x] Simple line-by-line diffs powered by the `imara-diff` crate.
+* **generic rename tracker to find renames and copies**
+    * [x] find by exact match
+    * [x] find by similarity check
+    * [ ] heuristics to find best candidate
+    * [ ] find by basename to help detecting simple moves
+    * [ ] caching of diffable data
+* **blob**
+    * [ ] use external command for diff generation
+    * [ ] worktree conversions
+    * [ ] `textconv` filters
+* [ ] working with hunks of data
 * [x] API documentation
-  * [ ] Examples
+    * [ ] Examples
 
 [gix-diff-performance]: https://github.com/Byron/gitoxide/discussions/74
 

crate-status.md

@@ -13,7 +13,7 @@ autotests = false
 [features]
 default = ["blob"]
 ## Enable diffing of blobs using imara-diff, which also allows for a generic rewrite tracking implementation.
-blob = ["dep:imara-diff"]
+blob = ["dep:imara-diff", "dep:gix-filter"]
 ## Data structures implement `serde::Serialize` and `serde::Deserialize`.
 serde = ["dep:serde", "gix-hash/serde", "gix-object/serde"]
 ## Make it possible to compile to the `wasm32-unknown-unknown` target.
@@ -25,6 +25,7 @@ doctest = false
 [dependencies]
 gix-hash = { version = "^0.13.1", path = "../gix-hash" }
 gix-object = { version = "^0.38.0", path = "../gix-object" }
+gix-filter = { version = "^0.6.0", path = "../gix-filter", optional = true }
 
 thiserror = "1.0.32"
 imara-diff = { version = "0.1.3", optional = true }

gix-diff/Cargo.toml

@@ -0,0 +1,133 @@
+//! For using text diffs, please have a look at the [`imara-diff` documentation](https://docs.rs/imara-diff),
+//! maintained by [Pascal Kuthe](https://github.com/pascalkuthe).
+pub use imara_diff::*;
+
+use bstr::BStr;
+use bstr::BString;
+
+/// Information about the diff performed to detect similarity.
+#[derive(Debug, Default, Clone, Copy, PartialEq, PartialOrd)]
+pub struct DiffLineStats {
+    /// The amount of lines to remove from the source to get to the destination.
+    pub removals: u32,
+    /// The amount of lines to add to the source to get to the destination.
+    pub insertions: u32,
+    /// The amount of lines of the previous state, in the source.
+    pub before: u32,
+    /// The amount of lines of the new state, in the destination.
+    pub after: u32,
+    /// A range from 0 to 1.0, where 1.0 is a perfect match and 0.5 is a similarity of 50%.
+    /// Similarity is the ratio between all lines in the previous blob and the current blob,
+    /// calculated as `(old_lines_count - new_lines_count) as f32 / old_lines_count.max(new_lines_count) as f32`.
+    pub similarity: f32,
+}
+
+/// A set of values to define how to diff something that is associated with it using `git-attributes`.
+///
+/// Some values are related to diffing, some are related to conversions.
+#[derive(Debug, Clone)]
+struct Driver {
+    /// The name of the driver, as referred to by `[diff "name"]` in the git configuration.
+    pub name: BString,
+    /// The per-driver algorithm to use.
+    pub algorithm: Option<Algorithm>,
+    /// The external filter program to call like `<binary_to_text_command> /path/to/blob` which outputs a textual version of the provided
+    /// binary file.
+    /// Note that it's invoked with a shell if arguments are given.
+    pub binary_to_text_command: Option<BString>,
+    /// `true` if this driver deals with binary files, which means that a `binary_to_text_command` should be used to convert binary
+    /// into a textual representation.
+    pub is_binary: bool,
+}
+
+/// A trait to help access the worktree version of a file, if present.
+pub trait ReadWorktreeBlob {
+    /// Write the contents of the file, executable or link (i.e. the link target itself) at
+    /// `rela_path` to the initially empty `buf`.
+    /// `is_source` is `true` if this is the blob for the source of a rewrite (copy or rename). Otherwise it is the
+    /// destination.
+    ///
+    /// Return `std::io::ErrorKind::NotFound` if the file is not available in the working tree, which will make the
+    /// implementation to extract it from the object database instead and convert it to its working-tree counterpart.
+    fn read_worktree_blob(&mut self, rela_path: &BStr, buf: &mut Vec<u8>, is_source: bool) -> std::io::Result<()>;
+}
+
+///
+pub mod platform {
+    use super::Algorithm;
+    use crate::blob::{Platform, ReadWorktreeBlob};
+
+    /// A conversion pipeline to take an object or path from what's stored in `git` to what can be diffed, while
+    /// following the guidance of git-attributes at the respective path to learn if diffing should happen or if
+    /// the content is considered binary.
+    ///
+    /// There are two different conversion flows, where the target of the flow is a buffer with diffable content:
+    ///
+    /// * `worktree on disk` -> `text conversion`
+    /// * `object` -> `worktree-filters` -> `text conversion`
+    ///
+    /// Based on whether or not [`ReadWorktreeBlob`] can find the file in question, we either read directly from disk
+    /// or transform from the object database.
+    // TODO: make public and put outside to driver, construct separately
+    pub(crate) struct FilterPipeline<ReadBlob> {
+        filter: gix_filter::Pipeline,
+        read_blob: ReadBlob,
+        drivers: Vec<super::Driver>,
+    }
+
+    #[derive(Default)]
+    pub(crate) struct Diffable {
+        /// `None` unless this instance is set to an object to diff.
+        id_and_mode: Option<(gix_hash::ObjectId, gix_object::tree::EntryMode)>,
+        /// The complete and fully transformed content to use for diffing, an allocation for reuse
+        /// between setting different source and destination objects.
+        buf: Vec<u8>,
+    }
+
+    /// Options for use in [Platform::new()].
+    #[derive(Default, Copy, Clone)]
+    pub struct Options {
+        /// The algorithm to use when diffing.
+        pub algorithm: Option<Algorithm>,
+
+        /// If `false`, default `false`, no diffing of binary files will be attempted even
+        /// if a [`binary_to_text_command`](Driver::binary_to_text_command) is set in the driver.
+        pub allow_binary_to_text_command: bool,
+    }
+    /// Lifecycle
+    impl<ReadBlob: ReadWorktreeBlob> Platform<ReadBlob> {
+        /// Create a new instance with `options`, a way to read worktree blobs with `read_blob` and `drivers`.
+        pub fn new(
+            options: Options,
+            read_blob: ReadBlob,
+            filter: gix_filter::Pipeline,
+            drivers: Vec<super::Driver>,
+        ) -> Self {
+            Platform {
+                opts: options,
+                old: Diffable::default(),
+                new: Diffable::default(),
+                filter: FilterPipeline {
+                    filter,
+                    read_blob,
+                    drivers,
+                },
+            }
+        }
+    }
+}
+
+/// A utility for performing a diff of two blobs, including flexible conversions, conversion-caching
+/// acquisition of diff information.
+/// Note that this instance will not call external filters as their output can't be known programmatically,
+/// but it allows to prepare their input if the caller wishes to perform this task.
+struct Platform<ReadBlob> {
+    opts: platform::Options,
+    /// The old version of a diff-able blob, if set.
+    old: platform::Diffable,
+    /// The new version of a diff-able blob, if set.
+    new: platform::Diffable,
+
+    /// A way to convert objects into a diff-able format.
+    filter: platform::FilterPipeline<ReadBlob>,
+}