Skip to content

Commit d2ecc83

Browse files
gitstergitster
committed
Merge branch 'hn/reftable' into pu
A new refs backend "reftable" to replace the traditional combination of packed-refs files and one-file-per-ref loose refs has been implemented and integrated for improved performance and atomicity. At v6. * hn/reftable: Reftable support for git-core Add reftable library refs: document how ref_iterator_advance_fn should handle symrefs create .git/refs in files-backend.c refs.h: clarify reflog iteration order
2 parents a19ddba + 892e423 commit d2ecc83

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+7407
-32
lines changed

‎Documentation/technical/repository-version.txt

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,3 +100,10 @@ If set, by default "git config" reads from both "config" and
100100
multiple working directory mode, "config" file is shared while
101101
"config.worktree" is per-working directory (i.e., it's in
102102
GIT_COMMON_DIR/worktrees/<id>/config.worktree)
103+
104+
==== `refStorage`
105+
106+
Specifies the file format for the ref database. Values are `files`
107+
(for the traditional packed + loose ref format) and `reftable` for the
108+
binary reftable format. See https://github.com/google/reftable for
109+
more information.

‎Makefile

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -817,6 +817,7 @@ TEST_SHELL_PATH = $(SHELL_PATH)
817817
LIB_FILE = libgit.a
818818
XDIFF_LIB = xdiff/lib.a
819819
VCSSVN_LIB = vcs-svn/lib.a
820+
REFTABLE_LIB = reftable/libreftable.a
820821

821822
GENERATED_H += config-list.h
822823
GENERATED_H += command-list.h
@@ -967,6 +968,7 @@ LIB_OBJS += ref-filter.o
967968
LIB_OBJS += reflog-walk.o
968969
LIB_OBJS += refs.o
969970
LIB_OBJS += refs/files-backend.o
971+
LIB_OBJS += refs/reftable-backend.o
970972
LIB_OBJS += refs/iterator.o
971973
LIB_OBJS += refs/packed-backend.o
972974
LIB_OBJS += refs/ref-cache.o
@@ -1171,7 +1173,7 @@ THIRD_PARTY_SOURCES += compat/regex/%
11711173
THIRD_PARTY_SOURCES += sha1collisiondetection/%
11721174
THIRD_PARTY_SOURCES += sha1dc/%
11731175

1174-
GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB)
1176+
GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB)
11751177
EXTLIBS =
11761178

11771179
GIT_USER_AGENT = git/$(GIT_VERSION)
@@ -2369,11 +2371,28 @@ VCSSVN_OBJS += vcs-svn/sliding_window.o
23692371
VCSSVN_OBJS += vcs-svn/svndiff.o
23702372
VCSSVN_OBJS += vcs-svn/svndump.o
23712373

2374+
REFTABLE_OBJS += reftable/basics.o
2375+
REFTABLE_OBJS += reftable/block.o
2376+
REFTABLE_OBJS += reftable/bytes.o
2377+
REFTABLE_OBJS += reftable/file.o
2378+
REFTABLE_OBJS += reftable/iter.o
2379+
REFTABLE_OBJS += reftable/merged.o
2380+
REFTABLE_OBJS += reftable/pq.o
2381+
REFTABLE_OBJS += reftable/reader.o
2382+
REFTABLE_OBJS += reftable/record.o
2383+
REFTABLE_OBJS += reftable/slice.o
2384+
REFTABLE_OBJS += reftable/stack.o
2385+
REFTABLE_OBJS += reftable/tree.o
2386+
REFTABLE_OBJS += reftable/writer.o
2387+
REFTABLE_OBJS += reftable/zlib-compat.o
2388+
2389+
23722390
TEST_OBJS := $(patsubst %$X,%.o,$(TEST_PROGRAMS)) $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS))
23732391
OBJECTS := $(LIB_OBJS) $(BUILTIN_OBJS) $(PROGRAM_OBJS) $(TEST_OBJS) \
23742392
$(XDIFF_OBJS) \
23752393
$(VCSSVN_OBJS) \
23762394
$(FUZZ_OBJS) \
2395+
$(REFTABLE_OBJS) \
23772396
common-main.o \
23782397
git.o
23792398
ifndef NO_CURL
@@ -2514,6 +2533,9 @@ $(XDIFF_LIB): $(XDIFF_OBJS)
25142533
$(VCSSVN_LIB): $(VCSSVN_OBJS)
25152534
$(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^
25162535

2536+
$(REFTABLE_LIB): $(REFTABLE_OBJS)
2537+
$(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^
2538+
25172539
export DEFAULT_EDITOR DEFAULT_PAGER
25182540

25192541
Documentation/GIT-EXCLUDED-PROGRAMS: FORCE

‎builtin/clone.c

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1097,7 +1097,9 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
10971097
}
10981098
}
10991099

1100-
init_db(git_dir, real_git_dir, option_template, INIT_DB_QUIET);
1100+
init_db(git_dir, real_git_dir, option_template,
1101+
DEFAULT_REF_STORAGE, /* XXX */
1102+
INIT_DB_QUIET);
11011103

11021104
if (real_git_dir)
11031105
git_dir = real_git_dir;

‎builtin/init-db.c

Lines changed: 40 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,8 @@ static int needs_work_tree_config(const char *git_dir, const char *work_tree)
177177
}
178178

179179
static int create_default_files(const char *template_path,
180-
const char *original_git_dir)
180+
const char *original_git_dir,
181+
const char *ref_storage_format, int flags)
181182
{
182183
struct stat st1;
183184
struct strbuf buf = STRBUF_INIT;
@@ -213,6 +214,7 @@ static int create_default_files(const char *template_path,
213214
is_bare_repository_cfg = init_is_bare_repository;
214215
if (init_shared_repository != -1)
215216
set_shared_repository(init_shared_repository);
217+
the_repository->ref_storage_format = xstrdup(ref_storage_format);
216218

217219
/*
218220
* We would have created the above under user's umask -- under
@@ -222,12 +224,19 @@ static int create_default_files(const char *template_path,
222224
adjust_shared_perm(get_git_dir());
223225
}
224226

227+
/*
228+
* Check to see if .git/HEAD exists; this must happen before
229+
* initializing the ref db, because we want to see if there is an
230+
* existing HEAD.
231+
*/
232+
path = git_path_buf(&buf, "HEAD");
233+
reinit = (!access(path, R_OK) ||
234+
readlink(path, junk, sizeof(junk) - 1) != -1);
235+
225236
/*
226237
* We need to create a "refs" dir in any case so that older
227238
* versions of git can tell that this is a repository.
228239
*/
229-
safe_create_dir(git_path("refs"), 1);
230-
adjust_shared_perm(git_path("refs"));
231240

232241
if (refs_init_db(&err))
233242
die("failed to set up refs db: %s", err.buf);
@@ -236,17 +245,21 @@ static int create_default_files(const char *template_path,
236245
* Create the default symlink from ".git/HEAD" to the "master"
237246
* branch, if it does not exist yet.
238247
*/
239-
path = git_path_buf(&buf, "HEAD");
240-
reinit = (!access(path, R_OK)
241-
|| readlink(path, junk, sizeof(junk)-1) != -1);
242248
if (!reinit) {
243249
if (create_symref("HEAD", "refs/heads/master", NULL) < 0)
244250
exit(1);
251+
} else {
252+
/*
253+
* XXX should check whether our ref backend matches the
254+
* original one; if not, either die() or convert
255+
*/
245256
}
246257

247258
/* This forces creation of new config file */
248-
xsnprintf(repo_version_string, sizeof(repo_version_string),
249-
"%d", GIT_REPO_VERSION);
259+
xsnprintf(repo_version_string, sizeof(repo_version_string), "%d",
260+
!strcmp(ref_storage_format, "reftable") ?
261+
GIT_REPO_VERSION_READ :
262+
GIT_REPO_VERSION);
250263
git_config_set("core.repositoryformatversion", repo_version_string);
251264

252265
/* Check filemode trustability */
@@ -341,7 +354,8 @@ static void separate_git_dir(const char *git_dir, const char *git_link)
341354
}
342355

343356
int init_db(const char *git_dir, const char *real_git_dir,
344-
const char *template_dir, unsigned int flags)
357+
const char *template_dir, const char *ref_storage_format,
358+
unsigned int flags)
345359
{
346360
int reinit;
347361
int exist_ok = flags & INIT_DB_EXIST_OK;
@@ -380,7 +394,8 @@ int init_db(const char *git_dir, const char *real_git_dir,
380394
*/
381395
check_repository_format();
382396

383-
reinit = create_default_files(template_dir, original_git_dir);
397+
reinit = create_default_files(template_dir, original_git_dir,
398+
ref_storage_format, flags);
384399

385400
create_object_directory();
386401

@@ -405,6 +420,8 @@ int init_db(const char *git_dir, const char *real_git_dir,
405420
git_config_set("receive.denyNonFastforwards", "true");
406421
}
407422

423+
git_config_set("extensions.refStorage", ref_storage_format);
424+
408425
if (!(flags & INIT_DB_QUIET)) {
409426
int len = strlen(git_dir);
410427

@@ -478,20 +495,24 @@ static const char *const init_db_usage[] = {
478495
int cmd_init_db(int argc, const char **argv, const char *prefix)
479496
{
480497
const char *git_dir;
498+
const char *ref_storage_format = DEFAULT_REF_STORAGE;
481499
const char *real_git_dir = NULL;
482500
const char *work_tree;
483501
const char *template_dir = NULL;
484502
unsigned int flags = 0;
485503
const struct option init_db_options[] = {
486-
OPT_STRING(0, "template", &template_dir, N_("template-directory"),
487-
N_("directory from which templates will be used")),
504+
OPT_STRING(0, "template", &template_dir,
505+
N_("template-directory"),
506+
N_("directory from which templates will be used")),
488507
OPT_SET_INT(0, "bare", &is_bare_repository_cfg,
489-
N_("create a bare repository"), 1),
508+
N_("create a bare repository"), 1),
490509
{ OPTION_CALLBACK, 0, "shared", &init_shared_repository,
491-
N_("permissions"),
492-
N_("specify that the git repository is to be shared amongst several users"),
493-
PARSE_OPT_OPTARG | PARSE_OPT_NONEG, shared_callback, 0},
510+
N_("permissions"),
511+
N_("specify that the git repository is to be shared amongst several users"),
512+
PARSE_OPT_OPTARG | PARSE_OPT_NONEG, shared_callback, 0 },
494513
OPT_BIT('q', "quiet", &flags, N_("be quiet"), INIT_DB_QUIET),
514+
OPT_STRING(0, "ref-storage", &ref_storage_format, N_("backend"),
515+
N_("the ref storage format to use")),
495516
OPT_STRING(0, "separate-git-dir", &real_git_dir, N_("gitdir"),
496517
N_("separate git dir from working tree")),
497518
OPT_END()
@@ -593,9 +614,11 @@ int cmd_init_db(int argc, const char **argv, const char *prefix)
593614
}
594615

595616
UNLEAK(real_git_dir);
617+
UNLEAK(ref_storage_format);
596618
UNLEAK(git_dir);
597619
UNLEAK(work_tree);
598620

599621
flags |= INIT_DB_EXIST_OK;
600-
return init_db(git_dir, real_git_dir, template_dir, flags);
622+
return init_db(git_dir, real_git_dir, template_dir, ref_storage_format,
623+
flags);
601624
}

‎cache.h

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -627,7 +627,8 @@ int path_inside_repo(const char *prefix, const char *path);
627627
#define INIT_DB_EXIST_OK 0x0002
628628

629629
int init_db(const char *git_dir, const char *real_git_dir,
630-
const char *template_dir, unsigned int flags);
630+
const char *template_dir, const char *ref_storage_format,
631+
unsigned int flags);
631632

632633
void sanitize_stdfds(void);
633634
int daemonize(void);
@@ -1041,6 +1042,7 @@ struct repository_format {
10411042
int is_bare;
10421043
int hash_algo;
10431044
char *work_tree;
1045+
char *ref_storage;
10441046
struct string_list unknown_extensions;
10451047
};
10461048

‎refs.c

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
/*
2121
* List of all available backends
2222
*/
23-
static struct ref_storage_be *refs_backends = &refs_be_files;
23+
static struct ref_storage_be *refs_backends = &refs_be_reftable;
2424

2525
static struct ref_storage_be *find_ref_storage_backend(const char *name)
2626
{
@@ -1836,13 +1836,13 @@ static struct ref_store *lookup_ref_store_map(struct hashmap *map,
18361836
* Create, record, and return a ref_store instance for the specified
18371837
* gitdir.
18381838
*/
1839-
static struct ref_store *ref_store_init(const char *gitdir,
1839+
static struct ref_store *ref_store_init(const char *gitdir, const char *be_name,
18401840
unsigned int flags)
18411841
{
1842-
const char *be_name = "files";
1843-
struct ref_storage_be *be = find_ref_storage_backend(be_name);
1842+
struct ref_storage_be *be;
18441843
struct ref_store *refs;
18451844

1845+
be = find_ref_storage_backend(be_name);
18461846
if (!be)
18471847
BUG("reference backend %s is unknown", be_name);
18481848

@@ -1858,7 +1858,10 @@ struct ref_store *get_main_ref_store(struct repository *r)
18581858
if (!r->gitdir)
18591859
BUG("attempting to get main_ref_store outside of repository");
18601860

1861-
r->refs = ref_store_init(r->gitdir, REF_STORE_ALL_CAPS);
1861+
r->refs = ref_store_init(r->gitdir,
1862+
r->ref_storage_format ? r->ref_storage_format :
1863+
DEFAULT_REF_STORAGE,
1864+
REF_STORE_ALL_CAPS);
18621865
return r->refs;
18631866
}
18641867

@@ -1913,7 +1916,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule)
19131916
goto done;
19141917

19151918
/* assume that add_submodule_odb() has been called */
1916-
refs = ref_store_init(submodule_sb.buf,
1919+
refs = ref_store_init(submodule_sb.buf, DEFAULT_REF_STORAGE, /* XXX */
19171920
REF_STORE_READ | REF_STORE_ODB);
19181921
register_ref_store_map(&submodule_ref_stores, "submodule",
19191922
refs, submodule);
@@ -1927,6 +1930,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule)
19271930

19281931
struct ref_store *get_worktree_ref_store(const struct worktree *wt)
19291932
{
1933+
const char *format = DEFAULT_REF_STORAGE; /* XXX */
19301934
struct ref_store *refs;
19311935
const char *id;
19321936

@@ -1940,9 +1944,9 @@ struct ref_store *get_worktree_ref_store(const struct worktree *wt)
19401944

19411945
if (wt->id)
19421946
refs = ref_store_init(git_common_path("worktrees/%s", wt->id),
1943-
REF_STORE_ALL_CAPS);
1947+
format, REF_STORE_ALL_CAPS);
19441948
else
1945-
refs = ref_store_init(get_git_common_dir(),
1949+
refs = ref_store_init(get_git_common_dir(), format,
19461950
REF_STORE_ALL_CAPS);
19471951

19481952
if (refs)

‎refs.h

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@ struct string_list;
99
struct string_list_item;
1010
struct worktree;
1111

12+
/* XXX where should this be? */
13+
#define DEFAULT_REF_STORAGE "files"
14+
1215
/*
1316
* Resolve a reference, recursively following symbolic refererences.
1417
*
@@ -444,18 +447,21 @@ int delete_refs(const char *msg, struct string_list *refnames,
444447
int refs_delete_reflog(struct ref_store *refs, const char *refname);
445448
int delete_reflog(const char *refname);
446449

447-
/* iterate over reflog entries */
450+
/* Iterate over reflog entries. */
448451
typedef int each_reflog_ent_fn(
449452
struct object_id *old_oid, struct object_id *new_oid,
450453
const char *committer, timestamp_t timestamp,
451454
int tz, const char *msg, void *cb_data);
452455

456+
/* Iterate in over reflog entries, oldest entry first. */
453457
int refs_for_each_reflog_ent(struct ref_store *refs, const char *refname,
454458
each_reflog_ent_fn fn, void *cb_data);
455459
int refs_for_each_reflog_ent_reverse(struct ref_store *refs,
456460
const char *refname,
457461
each_reflog_ent_fn fn,
458462
void *cb_data);
463+
464+
/* Call a function for each reflog entry, oldest entry first. */
459465
int for_each_reflog_ent(const char *refname, each_reflog_ent_fn fn, void *cb_data);
460466
int for_each_reflog_ent_reverse(const char *refname, each_reflog_ent_fn fn, void *cb_data);
461467

‎refs/files-backend.c

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3157,9 +3157,15 @@ static int files_init_db(struct ref_store *ref_store, struct strbuf *err)
31573157
files_downcast(ref_store, REF_STORE_WRITE, "init_db");
31583158
struct strbuf sb = STRBUF_INIT;
31593159

3160+
files_ref_path(refs, &sb, "refs");
3161+
safe_create_dir(sb.buf, 1);
3162+
/* adjust permissions even if directory already exists. */
3163+
adjust_shared_perm(sb.buf);
3164+
31603165
/*
31613166
* Create .git/refs/{heads,tags}
31623167
*/
3168+
strbuf_reset(&sb);
31633169
files_ref_path(refs, &sb, "refs/heads");
31643170
safe_create_dir(sb.buf, 1);
31653171

‎refs/refs-internal.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -438,6 +438,11 @@ void base_ref_iterator_free(struct ref_iterator *iter);
438438

439439
/* Virtual function declarations for ref_iterators: */
440440

441+
/*
442+
* backend-specific implementation of ref_iterator_advance.
443+
* For symrefs, the function should set REF_ISSYMREF, and it should also dereference
444+
* the symref to provide the OID referent.
445+
*/
441446
typedef int ref_iterator_advance_fn(struct ref_iterator *ref_iterator);
442447

443448
typedef int ref_iterator_peel_fn(struct ref_iterator *ref_iterator,
@@ -656,6 +661,7 @@ struct ref_storage_be {
656661
};
657662

658663
extern struct ref_storage_be refs_be_files;
664+
extern struct ref_storage_be refs_be_reftable;
659665
extern struct ref_storage_be refs_be_packed;
660666

661667
/*

0 commit comments

Comments
 (0)