PR 141948: Restructure fast checkout path to be closer to what we're trying to push upstream

 - Enable fast path when not in git root.
 - Enable calling show_local_changes at end of checkout when on fast path.

Author: Ben Peart

builtin/checkout.c

@@ -39,6 +39,10 @@ struct checkout_opts {
 	int ignore_skipworktree;
 	int ignore_other_worktrees;
 	int show_progress;
+	/*
+	 * If new checkout options are added, needs_working_tree_merge
+	 * should be updated accordingly.
+	 */
 
 	const char *new_branch;
 	const char *new_branch_force;
@@ -461,130 +465,222 @@ static void setup_branch_path(struct branch_info *branch)
 	branch->path = strbuf_detach(&buf, NULL);
 }
 
-static int merge_working_tree(const struct checkout_opts *opts,
-			      struct branch_info *old,
-			      struct branch_info *new,
-			      int *writeout_error)
+static int needs_working_tree_merge(const struct checkout_opts *opts,
+	const struct branch_info *old,
+	const struct branch_info *new)
 {
-	int ret;
-	struct lock_file *lock_file = xcalloc(1, sizeof(struct lock_file));
+	/*
+	 * We must do the merge if we are actually moving to a new
+	 * commit tree.
+	 */
+	if (!old->commit || !new->commit ||
+		oidcmp(&old->commit->tree->object.oid, &new->commit->tree->object.oid))
+		return 1;
 
-	hold_locked_index(lock_file, 1);
-	if (read_cache_preload(NULL) < 0)
-		return error(_("corrupt index file"));
+	/*
+	 * opts->patch_mode cannot be used with switching branches so is
+	 * not tested here
+	 */
 
-	resolve_undo_clear();
-	if (opts->force) {
-		ret = reset_tree(new->commit->tree, opts, 1, writeout_error);
-		if (ret)
-			return ret;
-	} else {
-		struct tree_desc trees[2];
-		struct tree *tree;
-		struct unpack_trees_options topts;
+	/*
+	 * opts->quiet only impacts output so doesn't require a merge
+	 */
 
-		memset(&topts, 0, sizeof(topts));
-		topts.head_idx = -1;
-		topts.src_index = &the_index;
-		topts.dst_index = &the_index;
+	/*
+	 * Honor the explicit request for a three-way merge or to throw away
+	 * local changes
+	 */
+	if (opts->merge || opts->force)
+		return 1;
 
-		setup_unpack_trees_porcelain(&topts, "checkout");
+	/*
+	 * --detach is documented as "updating the index and the files in the
+	 * working tree" but this optimization skips those steps so fall through
+	 * to the regular code path.
+	 */
+	if (opts->force_detach)
+		return 1;
 
-		refresh_cache(REFRESH_QUIET);
+	/*
+	 * opts->writeout_stage cannot be used with switching branches so is
+	 * not tested here
+	 */
 
-		if (unmerged_cache()) {
-			error(_("you need to resolve your current index first"));
-			return 1;
-		}
+	/*
+	 * Honor the explicit ignore requests
+	 */
+	if (!opts->overwrite_ignore || opts->ignore_skipworktree ||
+		opts->ignore_other_worktrees)
+		return 1;
 
-		/* 2-way merge to the new branch */
-		topts.initial_checkout = is_cache_unborn();
-		topts.update = 1;
-		topts.merge = 1;
-		topts.gently = opts->merge && old->commit;
-		topts.verbose_update = opts->show_progress;
-		topts.fn = twoway_merge;
-		if (opts->overwrite_ignore) {
-			topts.dir = xcalloc(1, sizeof(*topts.dir));
-			topts.dir->flags |= DIR_SHOW_IGNORED;
-			setup_standard_excludes(topts.dir);
-		}
-		tree = parse_tree_indirect(old->commit ?
-					   old->commit->object.oid.hash :
-					   EMPTY_TREE_SHA1_BIN);
-		init_tree_desc(&trees[0], tree->buffer, tree->size);
-		tree = parse_tree_indirect(new->commit->object.oid.hash);
-		init_tree_desc(&trees[1], tree->buffer, tree->size);
-
-		ret = unpack_trees(2, trees, &topts);
-		if (ret == -1) {
-			/*
-			 * Unpack couldn't do a trivial merge; either
-			 * give up or do a real merge, depending on
-			 * whether the merge flag was used.
-			 */
-			struct tree *result;
-			struct tree *work;
-			struct merge_options o;
-			if (!opts->merge)
-				return 1;
+	/*
+	 * opts->show_progress only impacts output so doesn't require a merge
+	 */
 
-			/*
-			 * Without old->commit, the below is the same as
-			 * the two-tree unpack we already tried and failed.
-			 */
-			if (!old->commit)
-				return 1;
+	/*
+	 * If we aren't creating a new branch any changes or updates will
+	 * happen in the existing branch.  Since that could only be updating
+	 * the index and working directory, we don't want to skip those steps
+	 * or we've defeated any purpose in running the command.
+	 */
+	if (!opts->new_branch)
+		return 1;
+
+	/*
+	 * new_branch_force is defined to "create/reset and checkout a branch"
+	 * so needs to go through the merge to do the reset
+	 */
+	if (opts->new_branch_force)
+		return 1;
 
-			/* Do more real merge */
+	/*
+	 * A new orphaned branch requrires the index and the working tree to be
+	 * adjusted to <start_point>
+	 */
+	if (opts->new_orphan_branch)
+		return 1;
 
-			/*
-			 * We update the index fully, then write the
-			 * tree from the index, then merge the new
-			 * branch with the current tree, with the old
-			 * branch as the base. Then we reset the index
-			 * (but not the working tree) to the new
-			 * branch, leaving the working tree as the
-			 * merged version, but skipping unmerged
-			 * entries in the index.
-			 */
+	/*
+	 * Remaining variables are not checkout options but used to track state
+	 */
 
-			add_files_to_cache(NULL, NULL, 0, 0);
-			/*
-			 * NEEDSWORK: carrying over local changes
-			 * when branches have different end-of-line
-			 * normalization (or clean+smudge rules) is
-			 * a pain; plumb in an option to set
-			 * o.renormalize?
-			 */
-			init_merge_options(&o);
-			o.verbosity = 0;
-			work = write_tree_from_memory(&o);
+	return 0;
+}
 
-			ret = reset_tree(new->commit->tree, opts, 1,
-					 writeout_error);
-			if (ret)
-				return ret;
-			o.ancestor = old->name;
-			o.branch1 = new->name;
-			o.branch2 = "local";
-			merge_trees(&o, new->commit->tree, work,
-				old->commit->tree, &result);
-			ret = reset_tree(new->commit->tree, opts, 0,
-					 writeout_error);
+
+static int merge_working_tree(const struct checkout_opts *opts,
+			      struct branch_info *old,
+			      struct branch_info *new,
+			      int *writeout_error)
+{
+	/*
+	 * Skip merging the trees, updating the index, and work tree only if we
+	 * are simply creating a new branch via "git checkout -b foo."  Any other
+	 * options or usage will continue to do all these steps.
+	 */
+	if (!gvfs_config_is_set(GVFS_SKIP_MERGE_IN_CHECKOUT) ||
+		needs_working_tree_merge(opts, old, new)) {
+
+		int ret;
+		struct lock_file *lock_file = xcalloc(1, sizeof(struct lock_file));
+
+		hold_locked_index(lock_file, 1);
+		if (read_cache_preload(NULL) < 0)
+			return error(_("corrupt index file"));
+
+		resolve_undo_clear();
+		if (opts->force) {
+			ret = reset_tree(new->commit->tree, opts, 1, writeout_error);
 			if (ret)
 				return ret;
+		} else {
+			struct tree_desc trees[2];
+			struct tree *tree;
+			struct unpack_trees_options topts;
+
+			memset(&topts, 0, sizeof(topts));
+			topts.head_idx = -1;
+			topts.src_index = &the_index;
+			topts.dst_index = &the_index;
+
+			setup_unpack_trees_porcelain(&topts, "checkout");
+
+			refresh_cache(REFRESH_QUIET);
+
+			if (unmerged_cache()) {
+				error(_("you need to resolve your current index first"));
+				return 1;
+			}
+
+			/* 2-way merge to the new branch */
+			topts.initial_checkout = is_cache_unborn();
+			topts.update = 1;
+			topts.merge = 1;
+			topts.gently = opts->merge && old->commit;
+			topts.verbose_update = opts->show_progress;
+			topts.fn = twoway_merge;
+			if (opts->overwrite_ignore) {
+				topts.dir = xcalloc(1, sizeof(*topts.dir));
+				topts.dir->flags |= DIR_SHOW_IGNORED;
+				setup_standard_excludes(topts.dir);
+			}
+			tree = parse_tree_indirect(old->commit ?
+							old->commit->object.oid.hash :
+							EMPTY_TREE_SHA1_BIN);
+			init_tree_desc(&trees[0], tree->buffer, tree->size);
+			tree = parse_tree_indirect(new->commit->object.oid.hash);
+			init_tree_desc(&trees[1], tree->buffer, tree->size);
+
+			ret = unpack_trees(2, trees, &topts);
+			if (ret == -1) {
+				/*
+				 * Unpack couldn't do a trivial merge; either
+				 * give up or do a real merge, depending on
+				 * whether the merge flag was used.
+				 */
+				struct tree *result;
+				struct tree *work;
+				struct merge_options o;
+				if (!opts->merge)
+					return 1;
+
+				/*
+				 * Without old->commit, the below is the same as
+				 * the two-tree unpack we already tried and failed.
+				 */
+				if (!old->commit)
+					return 1;
+
+				/* Do more real merge */
+
+				/*
+				 * We update the index fully, then write the
+				 * tree from the index, then merge the new
+				 * branch with the current tree, with the old
+				 * branch as the base. Then we reset the index
+				 * (but not the working tree) to the new
+				 * branch, leaving the working tree as the
+				 * merged version, but skipping unmerged
+				 * entries in the index.
+				 */
+
+				add_files_to_cache(NULL, NULL, 0, 0);
+				/*
+				 * NEEDSWORK: carrying over local changes
+				 * when branches have different end-of-line
+				 * normalization (or clean+smudge rules) is
+				 * a pain; plumb in an option to set
+				 * o.renormalize?
+				 */
+				init_merge_options(&o);
+				o.verbosity = 0;
+				work = write_tree_from_memory(&o);
+
+				ret = reset_tree(new->commit->tree, opts, 1,
+						writeout_error);
+				if (ret)
+					return ret;
+				o.ancestor = old->name;
+				o.branch1 = new->name;
+				o.branch2 = "local";
+				merge_trees(&o, new->commit->tree, work,
+					old->commit->tree, &result);
+				ret = reset_tree(new->commit->tree, opts, 0,
+						 writeout_error);
+				if (ret)
+					return ret;
+			}
 		}
-	}
 
-	if (!active_cache_tree)
-		active_cache_tree = cache_tree();
+		if (!active_cache_tree)
+			active_cache_tree = cache_tree();
 
-	if (!cache_tree_fully_valid(active_cache_tree))
-		cache_tree_update(&the_index, WRITE_TREE_SILENT | WRITE_TREE_REPAIR);
+		if (!cache_tree_fully_valid(active_cache_tree))
+			cache_tree_update(&the_index, WRITE_TREE_SILENT | WRITE_TREE_REPAIR);
 
-	if (write_locked_index(&the_index, lock_file, COMMIT_LOCK))
-		die(_("unable to write new index file"));
+		if (write_locked_index(&the_index, lock_file, COMMIT_LOCK))
+			die(_("unable to write new index file"));
+	}
 
 	if (!opts->force && !opts->quiet)
 		show_local_changes(&new->commit->object, &opts->diff_options);
@@ -825,26 +921,10 @@ static int switch_branches(const struct checkout_opts *opts,
 		parse_commit_or_die(new->commit);
 	}
 
-	/*
-	 * Optimize the performance of "git checkout foo" by skipping the call
-	 * to merge_working_tree. Make this as restrictive as possible, only
-	 * checkout a new branch with the current commit. Any other options force
-	 * it through the old path.
-	 */
-	if (!gvfs_config_is_set(GVFS_SKIP_MERGE_IN_CHECKOUT)
-		|| !old.commit || !new->commit
-		|| oidcmp(&old.commit->object.oid, &new->commit->object.oid)
-		|| !opts->new_branch || opts->new_branch_force || opts->new_orphan_branch
-		|| opts->patch_mode || opts->merge || opts->force || opts->force_detach
-		|| opts->writeout_stage || !opts->overwrite_ignore
-		|| opts->ignore_skipworktree || opts->ignore_other_worktrees
-		|| opts->new_branch_log || opts->branch_exists || opts->prefix
-		|| opts->source_tree) {
-		ret = merge_working_tree(opts, &old, new, &writeout_error);
-		if (ret) {
-			free(path_to_free);
-			return ret;
-		}
+	ret = merge_working_tree(opts, &old, new, &writeout_error);
+	if (ret) {
+		free(path_to_free);
+		return ret;
 	}
 
 	if (!opts->quiet && !old.path && old.commit && new->commit != old.commit)