Skip to content

Commit 83757c7

Browse files
KarthikNayakgitster
authored andcommitted
for-each-ref: introduce a '--skip-until' option
The `git-for-each-ref(1)` command is used to iterate over references present in a repository. In large repositories with millions of references, it would be optimal to paginate this output such that we can start iteration from a given reference. This would avoid having to iterate over all references from the beginning each time when paginating through results. The previous commit added 'seek' functionality to the reference backends. Utilize this and expose a '--skip-until' option in 'git-for-each-ref(1)'. When used, the reference iteration seeks to the first matching reference and iterates from there onward. This enables efficient pagination workflows like: git for-each-ref --count=100 git for-each-ref --count=100 --skip-until=refs/heads/branch-100 git for-each-ref --count=100 --skip-until=refs/heads/branch-200 Signed-off-by: Karthik Nayak <[email protected]> Signed-off-by: Junio C Hamano <[email protected]>
1 parent 44d49b2 commit 83757c7

File tree

5 files changed

+230
-19
lines changed

5 files changed

+230
-19
lines changed

Documentation/git-for-each-ref.adoc

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ SYNOPSIS
1414
[--points-at=<object>]
1515
[--merged[=<object>]] [--no-merged[=<object>]]
1616
[--contains[=<object>]] [--no-contains[=<object>]]
17-
[--exclude=<pattern> ...]
17+
[--exclude=<pattern> ...] [--skip-until=<pattern>]
1818

1919
DESCRIPTION
2020
-----------
@@ -108,6 +108,10 @@ TAB %(refname)`.
108108
--include-root-refs::
109109
List root refs (HEAD and pseudorefs) apart from regular refs.
110110

111+
--skip-until::
112+
Skip references up to the specified pattern. Cannot be used with
113+
general pattern matching.
114+
111115
FIELD NAMES
112116
-----------
113117

builtin/for-each-ref.c

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ static char const * const for_each_ref_usage[] = {
1313
N_("git for-each-ref [--points-at <object>]"),
1414
N_("git for-each-ref [--merged [<commit>]] [--no-merged [<commit>]]"),
1515
N_("git for-each-ref [--contains [<commit>]] [--no-contains [<commit>]]"),
16+
N_("git for-each-ref [--skip-until <pattern>]"),
1617
NULL
1718
};
1819

@@ -44,6 +45,7 @@ int cmd_for_each_ref(int argc,
4445
OPT_GROUP(""),
4546
OPT_INTEGER( 0 , "count", &format.array_opts.max_count, N_("show only <n> matched refs")),
4647
OPT_STRING( 0 , "format", &format.format, N_("format"), N_("format to use for the output")),
48+
OPT_STRING( 0 , "skip-until", &filter.seek, N_("skip-until"), N_("skip references until")),
4749
OPT__COLOR(&format.use_color, N_("respect format colors")),
4850
OPT_REF_FILTER_EXCLUDE(&filter),
4951
OPT_REF_SORT(&sorting_options),
@@ -100,6 +102,9 @@ int cmd_for_each_ref(int argc,
100102
filter.name_patterns = argv;
101103
}
102104

105+
if (filter.seek && filter.name_patterns && filter.name_patterns[0])
106+
die(_("cannot use --skip-until with patterns"));
107+
103108
if (include_root_refs)
104109
flags |= FILTER_REFS_ROOT_REFS | FILTER_REFS_DETACHED_HEAD;
105110

ref-filter.c

Lines changed: 39 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2692,10 +2692,13 @@ static int for_each_fullref_in_pattern(struct ref_filter *filter,
26922692
each_ref_fn cb,
26932693
void *cb_data)
26942694
{
2695+
struct ref_iterator *iter;
2696+
int flags = 0, ret = 0;
2697+
26952698
if (filter->kind & FILTER_REFS_ROOT_REFS) {
26962699
/* In this case, we want to print all refs including root refs. */
2697-
return refs_for_each_include_root_refs(get_main_ref_store(the_repository),
2698-
cb, cb_data);
2700+
flags |= DO_FOR_EACH_INCLUDE_ROOT_REFS;
2701+
goto non_prefix_iter;
26992702
}
27002703

27012704
if (!filter->match_as_path) {
@@ -2704,8 +2707,7 @@ static int for_each_fullref_in_pattern(struct ref_filter *filter,
27042707
* prefixes like "refs/heads/" etc. are stripped off,
27052708
* so we have to look at everything:
27062709
*/
2707-
return refs_for_each_fullref_in(get_main_ref_store(the_repository),
2708-
"", NULL, cb, cb_data);
2710+
goto non_prefix_iter;
27092711
}
27102712

27112713
if (filter->ignore_case) {
@@ -2714,20 +2716,28 @@ static int for_each_fullref_in_pattern(struct ref_filter *filter,
27142716
* so just return everything and let the caller
27152717
* sort it out.
27162718
*/
2717-
return refs_for_each_fullref_in(get_main_ref_store(the_repository),
2718-
"", NULL, cb, cb_data);
2719+
goto non_prefix_iter;
27192720
}
27202721

27212722
if (!filter->name_patterns[0]) {
27222723
/* no patterns; we have to look at everything */
2723-
return refs_for_each_fullref_in(get_main_ref_store(the_repository),
2724-
"", filter->exclude.v, cb, cb_data);
2724+
goto non_prefix_iter;
27252725
}
27262726

27272727
return refs_for_each_fullref_in_prefixes(get_main_ref_store(the_repository),
27282728
NULL, filter->name_patterns,
27292729
filter->exclude.v,
27302730
cb, cb_data);
2731+
2732+
non_prefix_iter:
2733+
iter = refs_ref_iterator_begin(get_main_ref_store(the_repository), "",
2734+
NULL, 0, flags);
2735+
if (filter->seek)
2736+
ret = ref_iterator_seek(iter, filter->seek, 0);
2737+
if (ret)
2738+
return ret;
2739+
2740+
return do_for_each_ref_iterator(iter, cb, cb_data);
27312741
}
27322742

27332743
/*
@@ -3200,26 +3210,37 @@ static int do_filter_refs(struct ref_filter *filter, unsigned int type, each_ref
32003210
if (!filter->kind)
32013211
die("filter_refs: invalid type");
32023212
else {
3213+
const char *prefix = NULL;
3214+
32033215
/*
32043216
* For common cases where we need only branches or remotes or tags,
32053217
* we only iterate through those refs. If a mix of refs is needed,
32063218
* we iterate over all refs and filter out required refs with the help
32073219
* of filter_ref_kind().
32083220
*/
32093221
if (filter->kind == FILTER_REFS_BRANCHES)
3210-
ret = refs_for_each_fullref_in(get_main_ref_store(the_repository),
3211-
"refs/heads/", NULL,
3212-
fn, cb_data);
3222+
prefix = "refs/heads/";
32133223
else if (filter->kind == FILTER_REFS_REMOTES)
3214-
ret = refs_for_each_fullref_in(get_main_ref_store(the_repository),
3215-
"refs/remotes/", NULL,
3216-
fn, cb_data);
3224+
prefix = "refs/remotes/";
32173225
else if (filter->kind == FILTER_REFS_TAGS)
3218-
ret = refs_for_each_fullref_in(get_main_ref_store(the_repository),
3219-
"refs/tags/", NULL, fn,
3220-
cb_data);
3221-
else if (filter->kind & FILTER_REFS_REGULAR)
3226+
prefix = "refs/tags/";
3227+
3228+
if (prefix) {
3229+
struct ref_iterator *iter;
3230+
3231+
iter = refs_ref_iterator_begin(get_main_ref_store(the_repository),
3232+
"", NULL, 0, 0);
3233+
3234+
if (filter->seek)
3235+
ret = ref_iterator_seek(iter, filter->seek, 0);
3236+
else if (prefix)
3237+
ret = ref_iterator_seek(iter, prefix, 1);
3238+
3239+
if (!ret)
3240+
ret = do_for_each_ref_iterator(iter, fn, cb_data);
3241+
} else if (filter->kind & FILTER_REFS_REGULAR) {
32223242
ret = for_each_fullref_in_pattern(filter, fn, cb_data);
3243+
}
32233244

32243245
/*
32253246
* When printing all ref types, HEAD is already included,

ref-filter.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ struct ref_array {
6464

6565
struct ref_filter {
6666
const char **name_patterns;
67+
const char *seek;
6768
struct strvec exclude;
6869
struct oid_array points_at;
6970
struct commit_list *with_commit;

t/t6302-for-each-ref-filter.sh

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -541,4 +541,184 @@ test_expect_success 'validate worktree atom' '
541541
test_cmp expect actual
542542
'
543543

544+
test_expect_success 'skip until with empty value' '
545+
cat >expect <<-\EOF &&
546+
refs/heads/main
547+
refs/heads/main_worktree
548+
refs/heads/side
549+
refs/odd/spot
550+
refs/tags/annotated-tag
551+
refs/tags/doubly-annotated-tag
552+
refs/tags/doubly-signed-tag
553+
refs/tags/foo1.10
554+
refs/tags/foo1.3
555+
refs/tags/foo1.6
556+
refs/tags/four
557+
refs/tags/one
558+
refs/tags/signed-tag
559+
refs/tags/three
560+
refs/tags/two
561+
EOF
562+
git for-each-ref --format="%(refname)" --skip-until="" >actual &&
563+
test_cmp expect actual
564+
'
565+
566+
test_expect_success 'skip until to a specific reference' '
567+
cat >expect <<-\EOF &&
568+
refs/odd/spot
569+
refs/tags/annotated-tag
570+
refs/tags/doubly-annotated-tag
571+
refs/tags/doubly-signed-tag
572+
refs/tags/foo1.10
573+
refs/tags/foo1.3
574+
refs/tags/foo1.6
575+
refs/tags/four
576+
refs/tags/one
577+
refs/tags/signed-tag
578+
refs/tags/three
579+
refs/tags/two
580+
EOF
581+
git for-each-ref --format="%(refname)" --skip-until=refs/odd/spot >actual &&
582+
test_cmp expect actual
583+
'
584+
585+
test_expect_success 'skip until to a specific reference with partial match' '
586+
cat >expect <<-\EOF &&
587+
refs/odd/spot
588+
refs/tags/annotated-tag
589+
refs/tags/doubly-annotated-tag
590+
refs/tags/doubly-signed-tag
591+
refs/tags/foo1.10
592+
refs/tags/foo1.3
593+
refs/tags/foo1.6
594+
refs/tags/four
595+
refs/tags/one
596+
refs/tags/signed-tag
597+
refs/tags/three
598+
refs/tags/two
599+
EOF
600+
git for-each-ref --format="%(refname)" --skip-until=refs/odd/sp >actual &&
601+
test_cmp expect actual
602+
'
603+
604+
test_expect_success 'skip until just behind a specific reference' '
605+
cat >expect <<-\EOF &&
606+
refs/odd/spot
607+
refs/tags/annotated-tag
608+
refs/tags/doubly-annotated-tag
609+
refs/tags/doubly-signed-tag
610+
refs/tags/foo1.10
611+
refs/tags/foo1.3
612+
refs/tags/foo1.6
613+
refs/tags/four
614+
refs/tags/one
615+
refs/tags/signed-tag
616+
refs/tags/three
617+
refs/tags/two
618+
EOF
619+
git for-each-ref --format="%(refname)" --skip-until=refs/odd/parrot >actual &&
620+
test_cmp expect actual
621+
'
622+
623+
test_expect_success 'skip until to specific directory' '
624+
cat >expect <<-\EOF &&
625+
refs/odd/spot
626+
refs/tags/annotated-tag
627+
refs/tags/doubly-annotated-tag
628+
refs/tags/doubly-signed-tag
629+
refs/tags/foo1.10
630+
refs/tags/foo1.3
631+
refs/tags/foo1.6
632+
refs/tags/four
633+
refs/tags/one
634+
refs/tags/signed-tag
635+
refs/tags/three
636+
refs/tags/two
637+
EOF
638+
git for-each-ref --format="%(refname)" --skip-until=refs/odd >actual &&
639+
test_cmp expect actual
640+
'
641+
642+
test_expect_success 'skip until to specific directory with trailing slash' '
643+
cat >expect <<-\EOF &&
644+
refs/odd/spot
645+
refs/tags/annotated-tag
646+
refs/tags/doubly-annotated-tag
647+
refs/tags/doubly-signed-tag
648+
refs/tags/foo1.10
649+
refs/tags/foo1.3
650+
refs/tags/foo1.6
651+
refs/tags/four
652+
refs/tags/one
653+
refs/tags/signed-tag
654+
refs/tags/three
655+
refs/tags/two
656+
EOF
657+
git for-each-ref --format="%(refname)" --skip-until=refs/lost >actual &&
658+
test_cmp expect actual
659+
'
660+
661+
test_expect_success 'skip until just behind a specific directory' '
662+
cat >expect <<-\EOF &&
663+
refs/odd/spot
664+
refs/tags/annotated-tag
665+
refs/tags/doubly-annotated-tag
666+
refs/tags/doubly-signed-tag
667+
refs/tags/foo1.10
668+
refs/tags/foo1.3
669+
refs/tags/foo1.6
670+
refs/tags/four
671+
refs/tags/one
672+
refs/tags/signed-tag
673+
refs/tags/three
674+
refs/tags/two
675+
EOF
676+
git for-each-ref --format="%(refname)" --skip-until=refs/odd/ >actual &&
677+
test_cmp expect actual
678+
'
679+
680+
test_expect_success 'skip until overflow specific reference length' '
681+
cat >expect <<-\EOF &&
682+
refs/tags/annotated-tag
683+
refs/tags/doubly-annotated-tag
684+
refs/tags/doubly-signed-tag
685+
refs/tags/foo1.10
686+
refs/tags/foo1.3
687+
refs/tags/foo1.6
688+
refs/tags/four
689+
refs/tags/one
690+
refs/tags/signed-tag
691+
refs/tags/three
692+
refs/tags/two
693+
EOF
694+
git for-each-ref --format="%(refname)" --skip-until=refs/odd/spotnew >actual &&
695+
test_cmp expect actual
696+
'
697+
698+
test_expect_success 'skip until overflow specific reference path' '
699+
cat >expect <<-\EOF &&
700+
refs/tags/annotated-tag
701+
refs/tags/doubly-annotated-tag
702+
refs/tags/doubly-signed-tag
703+
refs/tags/foo1.10
704+
refs/tags/foo1.3
705+
refs/tags/foo1.6
706+
refs/tags/four
707+
refs/tags/one
708+
refs/tags/signed-tag
709+
refs/tags/three
710+
refs/tags/two
711+
EOF
712+
git for-each-ref --format="%(refname)" --skip-until=refs/odd/spot/new >actual &&
713+
test_cmp expect actual
714+
'
715+
716+
test_expect_success 'skip until used with a pattern' '
717+
cat >expect <<-\EOF &&
718+
fatal: cannot use --skip-until with patterns
719+
EOF
720+
test_must_fail git for-each-ref --format="%(refname)" --skip-until=refs/odd/spot refs/tags 2>actual &&
721+
test_cmp expect actual
722+
'
723+
544724
test_done

0 commit comments

Comments
 (0)