Merge branch 'dirwalk'

Author: Byron

.gitignore

@@ -6,9 +6,15 @@ target/
 
 # repositories used for local testing
 /tests/fixtures/repos
+$/tests/fixtures/repos/
+
 /tests/fixtures/commit-graphs/
+$/tests/fixtures/commit-graphs/
 
 **/generated-do-not-edit/
 
 # Cargo lock files of fuzz targets - let's have the latest versions of everything under test
 **/fuzz/Cargo.lock
+
+# newer Git sees these as precious, older Git falls through to the pattern above
+$**/fuzz/Cargo.lock

Cargo.lock

@@ -129,7 +129,7 @@ prodash-render-line = ["prodash/render-line", "prodash-render-line-crossterm", "
 cache-efficiency-debug = ["gix-features/cache-efficiency-debug"]
 
 ##  A way to enable most `gitoxide-core` tools found in `ein tools`, namely `organize` and `estimate hours`.
-gitoxide-core-tools = ["gitoxide-core/organize", "gitoxide-core/estimate-hours", "gitoxide-core-tools-archive"]
+gitoxide-core-tools = ["gitoxide-core/organize", "gitoxide-core/estimate-hours", "gitoxide-core-tools-archive", "gitoxide-core-tools-clean"]
 
 ## A program to perform analytics on a `git` repository, using an auto-maintained sqlite database
 gitoxide-core-tools-query = ["gitoxide-core/query"]
@@ -140,6 +140,9 @@ gitoxide-core-tools-corpus = ["gitoxide-core/corpus"]
 ## A sub-command to generate archive from virtual worktree checkouts.
 gitoxide-core-tools-archive = ["gitoxide-core/archive"]
 
+## A sub-command to clean the worktree from untracked and ignored files.
+gitoxide-core-tools-clean = ["gitoxide-core/clean"]
+
 #! ### Building Blocks for mutually exclusive networking
 #! Blocking and async features are mutually exclusive and cause a compile-time error. This also means that `cargo … --all-features` will fail.
 #! Within each section, features can be combined.

Cargo.toml

@@ -309,3 +309,5 @@ GIT_SSH_COMMAND="ssh -VVV" \
 git <command>
 ```
 
+Consider adding `GIT_TRACE2_PERF=1` (possibly add `GIT_TRACE2_PERF_BRIEF=1` for brevity) as well for statistics and variables 
+(see [their source for more](https://github.com/git/git/blob/b50a608ba20348cb3dfc16a696816d51780e3f0f/trace2/tr2_sysenv.c#L50).

DEVELOPMENT.md

@@ -612,12 +612,12 @@ A plumbing crate with shared functionality regarding EWAH compressed bitmaps, as
 
 A git directory walk.
 
-* [ ] list untracked files
-    - [ ] `normal` - files and directories
-    - [ ] `all` - expand to untracked files in untracked directories
-* [ ] list ignored files
-    - [ ] `matching` mode (show every ignored file, do not aggregate into parent directory)
-    - [ ] `traditional` mode (aggregate all ignored files of a folder into ignoring the folder itself)
+* [x] list untracked files
+* [x] list ignored files
+* [x] collapsing of untracked and ignored directories
+* [x] pathspec based filtering
+* [ ] multi-threaded initialization of icase hash table is always used to accelerate index lookups, even if ignoreCase = false for performance
+* [ ] special handling of submodules (for now, submodules or nested repositories are detected, but they can't be walked into naturally)
 * [ ] accelerated walk with `untracked`-cache (as provided by `UNTR` extension of `gix_index::File`)
 
 ### gix-index
@@ -628,6 +628,7 @@ The git staging area.
   * [x] V2 - the default, including long-paths support
   * [x] V3 - extended flags
   * [x] V4 - delta-compression for paths
+  * [ ] TODO(perf): multi-threaded implementation should boost performance, spends most time in storing paths, has barely any benefit right now.
   * optional threading
     * [x] concurrent loading of index extensions
     * [x] threaded entry reading
@@ -658,7 +659,9 @@ The git staging area.
 * `stat` update
     * [ ] optional threaded `stat` based on thread_cost (aka preload)
 * [x] handling of `.gitignore` and system file exclude configuration
-* [ ] handle potential races
+* [x] lookups that ignore the case
+    * [ ] multi-threaded lookup table generation with the same algorithm as the one used by Git
+    * [ ] expand sparse folders (don't know how this relates to traversals right now)
 * maintain extensions when altering the cache
     * [ ] TREE for speeding up tree generation
     * [ ] REUC resolving undo

crate-status.md

@@ -28,6 +28,9 @@ corpus = [ "dep:rusqlite", "dep:sysinfo", "organize", "dep:crossbeam-channel", "
 ## The ability to create archives from virtual worktrees, similar to `git archive`.
 archive = ["dep:gix-archive-for-configuration-only", "gix/worktree-archive"]
 
+## The ability to clean a repository, similar to `git clean`.
+clean = [ "gix/dirwalk" ]
+
 #! ### Mutually Exclusive Networking
 #! If both are set, _blocking-client_ will take precedence, allowing `--all-features` to be used.
 

gitoxide-core/Cargo.toml

@@ -101,6 +101,12 @@ mod serde_only {
                     if f.fs_monitor().is_some() {
                         names.push("fs-monitor (FSMN)");
                     };
+                    if f.had_offset_table() {
+                        names.push("offset-table (IEOT)")
+                    }
+                    if f.had_end_of_index_marker() {
+                        names.push("end-of-index (EOIE)")
+                    }
                     Extensions { names, tree }
                 },
                 entries: {