add precompose_unicode related tests

* reading precomposes unicode
* writing precomposes unicode of input names for good measure

Author: Byron

Cargo.lock

@@ -25,6 +25,7 @@ gix-path = { version = "^0.10.3", path = "../gix-path" }
 gix-hash = { version = "^0.14.1", path = "../gix-hash" }
 gix-date = { version = "^0.8.3", path = "../gix-date" }
 gix-object = { version = "^0.40.1", path = "../gix-object" }
+gix-utils = { version = "^0.1.8", path = "../gix-utils" }
 gix-validate = { version = "^0.8.3", path = "../gix-validate" }
 gix-actor = { version = "^0.29.1", path = "../gix-actor" }
 gix-lock = { version = "^12.0.0", path = "../gix-lock" }

gix-ref/Cargo.toml

@@ -10,7 +10,7 @@ pub use error::Error;
 use crate::{
     file,
     store_impl::{file::loose, packed},
-    BStr, BString, FullNameRef, PartialNameRef, Reference,
+    BStr, BString, FullNameRef, PartialName, PartialNameRef, Reference,
 };
 
 impl file::Store {
@@ -66,27 +66,58 @@ impl file::Store {
         partial_name: &PartialNameRef,
         packed: Option<&packed::Buffer>,
     ) -> Result<Option<Reference>, Error> {
+        fn decompose_if(mut r: Reference, input_changed_to_precomposed: bool) -> Reference {
+            if input_changed_to_precomposed {
+                use gix_object::bstr::ByteSlice;
+                let decomposed = r
+                    .name
+                    .0
+                    .to_str()
+                    .ok()
+                    .map(|name| gix_utils::str::decompose(name.into()));
+                if let Some(Cow::Owned(decomposed)) = decomposed {
+                    r.name.0 = decomposed.into();
+                }
+            }
+            r
+        }
         let mut buf = BString::default();
+        let mut precomposed_partial_name_storage = packed.filter(|_| self.precompose_unicode).and_then(|_| {
+            use gix_object::bstr::ByteSlice;
+            let precomposed = partial_name.0.to_str().ok()?;
+            let precomposed = gix_utils::str::precompose(precomposed.into());
+            match precomposed {
+                Cow::Owned(precomposed) => Some(PartialName(precomposed.into())),
+                Cow::Borrowed(_) => None,
+            }
+        });
+        let precomposed_partial_name = precomposed_partial_name_storage.as_ref().map(|n| n.as_ref());
         for inbetween in &["", "tags", "heads", "remotes"] {
-            match self.find_inner(inbetween, partial_name, packed, &mut buf) {
-                Ok(Some(r)) => return Ok(Some(r)),
+            match self.find_inner(inbetween, partial_name, precomposed_partial_name, packed, &mut buf) {
+                Ok(Some(r)) => return Ok(Some(decompose_if(r, precomposed_partial_name.is_some()))),
                 Ok(None) => {
                     continue;
                 }
                 Err(err) => return Err(err),
             }
         }
         if partial_name.as_bstr() != "HEAD" {
+            if let Some(mut precomposed) = precomposed_partial_name_storage {
+                precomposed = precomposed.join("HEAD".into()).expect("HEAD is valid name");
+                precomposed_partial_name_storage = Some(precomposed);
+            }
             self.find_inner(
                 "remotes",
                 partial_name
                     .to_owned()
                     .join("HEAD".into())
                     .expect("HEAD is valid name")
                     .as_ref(),
+                precomposed_partial_name_storage.as_ref().map(|n| n.as_ref()),
                 None,
                 &mut buf,
             )
+            .map(|res| res.map(|r| decompose_if(r, precomposed_partial_name_storage.is_some())))
         } else {
             Ok(None)
         }
@@ -96,10 +127,13 @@ impl file::Store {
         &self,
         inbetween: &str,
         partial_name: &PartialNameRef,
+        precomposed_partial_name: Option<&PartialNameRef>,
         packed: Option<&packed::Buffer>,
         path_buf: &mut BString,
     ) -> Result<Option<Reference>, Error> {
-        let full_name = partial_name.construct_full_name_ref(inbetween, path_buf);
+        let full_name = precomposed_partial_name
+            .unwrap_or(partial_name)
+            .construct_full_name_ref(inbetween, path_buf);
         let content_buf = self.ref_contents(full_name).map_err(|err| Error::ReadFileContents {
             source: err,
             path: self.reference_path(full_name),

gix-ref/src/store/file/find.rs

@@ -16,6 +16,7 @@ impl file::Store {
         Ok(packed::Transaction::new_from_pack_and_lock(
             self.assure_packed_refs_uptodate()?,
             lock,
+            self.precompose_unicode,
         ))
     }
 

gix-ref/src/store/file/packed.rs

@@ -11,14 +11,21 @@ use crate::{
 /// How to handle packed refs during a transaction
 #[derive(Default)]
 pub enum PackedRefs<'a> {
-    /// Only propagate deletions of references. This is the default
+    /// Only propagate deletions of references. This is the default.
+    /// This means deleted references are removed from disk if they are loose and from the packed-refs file if they are present.
     #[default]
     DeletionsOnly,
-    /// Propagate deletions as well as updates to references which are peeled, that is contain an object id
+    /// Propagate deletions as well as updates to references which are peeled and contain an object id.
+    ///
+    /// This means deleted references are removed from disk if they are loose and from the packed-refs file if they are present,
+    /// while updates are also written into the loose file as well as into packed-refs, potentially creating an entry.
     DeletionsAndNonSymbolicUpdates(Box<dyn gix_object::Find + 'a>),
-    /// Propagate deletions as well as updates to references which are peeled, that is contain an object id. Furthermore delete the
+    /// Propagate deletions as well as updates to references which are peeled and contain an object id. Furthermore delete the
     /// reference which is originally updated if it exists. If it doesn't, the new value will be written into the packed ref right away.
     /// Note that this doesn't affect symbolic references at all, which can't be placed into packed refs.
+    ///
+    /// Thus, this is similar to `DeletionsAndNonSymbolicUpdates`, but removes the loose reference after the update, leaving only their copy
+    /// in `packed-refs`.
     DeletionsAndNonSymbolicUpdatesRemoveLooseSourceReference(Box<dyn gix_object::Find + 'a>),
 }
 

gix-ref/src/store/file/transaction/mod.rs

@@ -329,7 +329,7 @@ impl<'s, 'p> Transaction<'s, 'p> {
                         self.store
                             .assure_packed_refs_uptodate()?
                             .map(|p| {
-                                buffer_into_transaction(p, packed_refs_lock_fail_mode)
+                                buffer_into_transaction(p, packed_refs_lock_fail_mode, self.store.precompose_unicode)
                                     .map_err(Error::PackedTransactionAcquire)
                             })
                             .transpose()?

gix-ref/src/store/file/transaction/prepare.rs

@@ -38,6 +38,7 @@ pub(crate) struct Transaction {
     lock: Option<gix_lock::File>,
     #[allow(dead_code)] // It just has to be kept alive, hence no reads
     closed_lock: Option<gix_lock::Marker>,
+    precompose_unicode: bool,
 }
 
 /// A reference as parsed from the `packed-refs` file

gix-ref/src/store/packed/mod.rs

@@ -1,3 +1,4 @@
+use std::borrow::Cow;
 use std::{fmt::Formatter, io::Write};
 
 use crate::{
@@ -14,12 +15,14 @@ impl packed::Transaction {
     pub(crate) fn new_from_pack_and_lock(
         buffer: Option<file::packed::SharedBufferSnapshot>,
         lock: gix_lock::File,
+        precompose_unicode: bool,
     ) -> Self {
         packed::Transaction {
             buffer,
             edits: None,
             lock: Some(lock),
             closed_lock: None,
+            precompose_unicode,
         }
     }
 }
@@ -55,6 +58,26 @@ impl packed::Transaction {
         // Remove all edits which are deletions that aren't here in the first place
         let mut edits: Vec<Edit> = edits
             .into_iter()
+            .map(|mut edit| {
+                use gix_object::bstr::ByteSlice;
+                if self.precompose_unicode {
+                    let precomposed = edit
+                        .name
+                        .0
+                        .to_str()
+                        .ok()
+                        .map(|name| gix_utils::str::precompose(name.into()));
+                    match precomposed {
+                        None | Some(Cow::Borrowed(_)) => edit,
+                        Some(Cow::Owned(precomposed)) => {
+                            edit.name.0 = precomposed.into();
+                            edit
+                        }
+                    }
+                } else {
+                    edit
+                }
+            })
             .filter(|edit| {
                 if let Change::Delete { .. } = edit.change {
                     buffer.as_ref().map_or(true, |b| b.find(edit.name.as_ref()).is_ok())
@@ -227,13 +250,15 @@ fn write_edit(out: &mut dyn std::io::Write, edit: &Edit, lines_written: &mut i32
 pub(crate) fn buffer_into_transaction(
     buffer: file::packed::SharedBufferSnapshot,
     lock_mode: gix_lock::acquire::Fail,
+    precompose_unicode: bool,
 ) -> Result<packed::Transaction, gix_lock::acquire::Error> {
     let lock = gix_lock::File::acquire_to_update_resource(&buffer.path, lock_mode, None)?;
     Ok(packed::Transaction {
         buffer: Some(buffer),
         lock: Some(lock),
         closed_lock: None,
         edits: None,
+        precompose_unicode,
     })
 }
 

gix-ref/src/store/packed/transaction.rs

@@ -1,4 +1,157 @@
+use crate::file::transaction::prepare_and_commit::{committer, create_at};
+use crate::file::EmptyCommit;
+use gix_lock::acquire::Fail;
+use gix_ref::file::transaction::PackedRefs;
+use gix_ref::store::WriteReflog;
+use gix_ref::transaction::{Change, LogChange, PreviousValue, RefEdit};
+
 mod access;
 mod find;
 mod iter;
 mod reflog;
+
+#[test]
+fn precompose_unicode_journey() -> crate::Result {
+    let tmp = gix_testtools::tempfile::TempDir::new()?;
+    let precomposed_a = "ä";
+    let decomposed_a = "a\u{308}";
+    let root = tmp.path().join(decomposed_a);
+    std::fs::create_dir(&root)?;
+
+    let store_decomposed = gix_ref::file::Store::at(
+        root,
+        WriteReflog::Always,
+        gix_hash::Kind::Sha1,
+        false, /* precompose_unicode */
+    );
+    assert!(!store_decomposed.precompose_unicode);
+
+    let decomposed_ref = format!("refs/heads/{decomposed_a}");
+    store_decomposed
+        .transaction()
+        .prepare(Some(create_at(&decomposed_ref)), Fail::Immediately, Fail::Immediately)?
+        .commit(committer().to_ref())?;
+
+    let r = store_decomposed.iter()?.all()?.next().expect("created one ref")?;
+    assert_eq!(r.name.as_bstr(), decomposed_ref, "no transformation happens by default");
+
+    let fs_folds_decomposed = gix_fs::Capabilities::probe(tmp.path()).precompose_unicode;
+    if !fs_folds_decomposed {
+        // For file-access to work, we need a filesystem for which precomposed == decomposed.
+        return Ok(());
+    }
+
+    let store_precomposed = gix_ref::file::Store::at(
+        tmp.path().join(precomposed_a), // it's important that root paths are also precomposed then.
+        WriteReflog::Always,
+        gix_hash::Kind::Sha1,
+        true, /* precompose_unicode */
+    );
+
+    let precomposed_ref = format!("refs/heads/{precomposed_a}");
+    let r = store_precomposed.iter()?.all()?.next().expect("created one ref")?;
+    assert_eq!(
+        r.name.as_bstr(),
+        precomposed_ref,
+        "it transforms all refs it sees to precomposed format, important when sending them over the wire"
+    );
+
+    assert_eq!(
+        store_precomposed.find(precomposed_a)?.name.as_bstr(),
+        precomposed_ref,
+        "can find as precomposed, even though on disk is decomposed it is decomposed"
+    );
+    assert_eq!(
+        store_precomposed.find(decomposed_a)?.name.as_bstr(),
+        decomposed_ref,
+        "can find as decomposed, and it keeps it as is to not violate expectations of the returned name being equal to the input (when comparing as bytes)"
+    );
+
+    let decomposed_u = "u\u{308}";
+    let decomposed_ref = format!("refs/heads/{decomposed_u}");
+    let edits = store_precomposed
+        .transaction()
+        .prepare(Some(create_at(&decomposed_ref)), Fail::Immediately, Fail::Immediately)?
+        .commit(committer().to_ref())?;
+    assert_eq!(
+        edits[0].name.as_bstr(),
+        decomposed_ref,
+        "it doesn't alter the composition style to allow input and output to remain unchanged"
+    );
+
+    assert_eq!(
+        store_decomposed.iter()?.all()?.nth(1).expect("two refs")?.name.shorten(),
+        decomposed_u,
+        "the ref name isn't transformed in any way and left decomposed on disk as well, making sure internal loose/packed-ref interactions work reliably"
+    );
+
+    assert!(
+        store_precomposed.cached_packed_buffer()?.is_none(),
+        "no packed-refs yet"
+    );
+    let edits = store_precomposed
+        .transaction()
+        .packed_refs(PackedRefs::DeletionsAndNonSymbolicUpdatesRemoveLooseSourceReference(
+            Box::new(EmptyCommit),
+        ))
+        .prepare(
+            // Intentionally use the decomposed versions of their names
+            store_decomposed
+                .loose_iter()?
+                .filter_map(|r| r.ok().filter(|r| r.kind() == gix_ref::Kind::Peeled))
+                .map(|r| RefEdit {
+                    change: Change::Update {
+                        log: LogChange::default(),
+                        expected: PreviousValue::MustExistAndMatch(r.target.clone()),
+                        new: r.target,
+                    },
+                    name: r.name,
+                    deref: false,
+                }),
+            Fail::Immediately,
+            Fail::Immediately,
+        )?
+        .commit(committer().to_ref())?;
+    assert!(
+        store_precomposed.cached_packed_buffer()?.is_some(),
+        "refs were written into the packed-refs file"
+    );
+
+    assert_eq!(store_precomposed.loose_iter()?.count(), 0, "all loose refs are gone");
+    assert_eq!(edits.len(), 2);
+    assert_eq!(
+        edits[0].name.shorten(),
+        decomposed_a,
+        "composition stays the same for consistency"
+    );
+    assert_eq!(
+        edits[1].name.shorten(),
+        decomposed_u,
+        "composition stays the same for consistency"
+    );
+
+    assert_eq!(
+        store_decomposed.find(precomposed_a)?.name.shorten(),
+        precomposed_a,
+        "the decomposed store can only find what's in packed-refs verbatim"
+    );
+    assert!(
+        store_decomposed.try_find(decomposed_a)?.is_none(),
+        "decomposed inputs don't match in packed-refs"
+    );
+    assert_eq!(
+        store_precomposed.find(precomposed_a)?.name.shorten(),
+        precomposed_a,
+        "we find what's in the packed-refs file, which is native to packed-refs"
+    );
+    assert_eq!(
+        store_precomposed.find(decomposed_a)?.name.shorten(),
+        decomposed_a,
+        "despite the input being decomposed, we find the ref (in packed-refs) as precomposed, but return it just like we inserted it"
+    );
+
+    // TODO: symrefs
+    // TODO: namespace
+
+    Ok(())
+}

gix-ref/tests/file/store/mod.rs

@@ -54,7 +54,7 @@ pub(crate) mod prepare_and_commit {
         }
     }
 
-    fn create_at(name: &str) -> RefEdit {
+    pub(crate) fn create_at(name: &str) -> RefEdit {
         RefEdit {
             change: Change::Update {
                 log: LogChange {