read-cache: run verify_hdr() in background thread

This is a performance optimization.

Teach do_read_index() to call verify_hdr() using a thread
and allow SHA1 verification to run concurrently with the
parsing of index-entries and extensions.

For large index files, this cuts the startup time in half.

Signed-off-by: Jeff Hostetler <[email protected]>

Author: jeffhostetler

read-cache.c

@@ -21,6 +21,10 @@
 #include "utf8.h"
 #include "fsmonitor.h"
 
+#ifndef NO_PTHREADS
+#include <pthread.h>
+#endif
+
 /* Mask for the name length in ce_flags in the on-disk index */
 
 #define CE_NAMEMASK  (0x0fff)
@@ -1566,6 +1570,34 @@ static int verify_hdr(struct cache_header *hdr, unsigned long size)
 	return 0;
 }
 
+#ifndef NO_PTHREADS
+/*
+ * Require index file to be larger than this threshold before
+ * we bother using a thread to verify the SHA.
+ * This value was arbitrarily chosen.
+ */
+#define VERIFY_HDR_THRESHOLD	10*1024*1024
+
+struct verify_hdr_thread_data
+{
+	pthread_t thread_id;
+	struct cache_header *hdr;
+	size_t size;
+	int result;
+};
+
+/*
+ * A thread proc to run the verify_hdr() computation
+ * in a background thread.
+ */
+static void *verify_hdr_thread(void *_data)
+{
+	struct verify_hdr_thread_data *p = _data;
+	p->result = verify_hdr(p->hdr, (unsigned long)p->size);
+	return NULL;
+}
+#endif
+
 static int read_index_extension(struct index_state *istate,
 				const char *ext, void *data, unsigned long sz)
 {
@@ -1774,6 +1806,9 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist)
 	void *mmap;
 	size_t mmap_size;
 	struct strbuf previous_name_buf = STRBUF_INIT, *previous_name;
+#ifndef NO_PTHREADS
+	struct verify_hdr_thread_data verify_hdr_thread_data;
+#endif
 
 	if (istate->initialized)
 		return istate->cache_nr;
@@ -1800,8 +1835,23 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist)
 	close(fd);
 
 	hdr = mmap;
+#ifdef NO_PTHREADS
 	if (verify_hdr(hdr, mmap_size) < 0)
 		goto unmap;
+#else
+	if (mmap_size < VERIFY_HDR_THRESHOLD) {
+		if (verify_hdr(hdr, mmap_size) < 0)
+			goto unmap;
+	} else {
+		verify_hdr_thread_data.hdr = hdr;
+		verify_hdr_thread_data.size = mmap_size;
+		verify_hdr_thread_data.result = -1;
+		if (pthread_create(
+				&verify_hdr_thread_data.thread_id, NULL,
+				verify_hdr_thread, &verify_hdr_thread_data))
+			die_errno("unable to start verify_hdr_thread");
+	}
+#endif
 
 	hashcpy(istate->sha1, (const unsigned char *)hdr + mmap_size - 20);
 	istate->version = ntohl(hdr->hdr_version);
@@ -1849,6 +1899,16 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist)
 		src_offset += 8;
 		src_offset += extsize;
 	}
+
+#ifndef NO_PTHREADS
+	if (mmap_size >= VERIFY_HDR_THRESHOLD) {
+		if (pthread_join(verify_hdr_thread_data.thread_id, NULL))
+			die_errno("unable to join verify_hdr_thread");
+		if (verify_hdr_thread_data.result < 0)
+			goto unmap;
+	}
+#endif
+
 	munmap(mmap, mmap_size);
 	return istate->cache_nr;