Merge branch 'busybox-w32'

This topic branch brings slightly experimental changes supporting Git
for Windows to use BusyBox-w32 to execute its shell scripts as well as
its test suite.

The test suite can be run by installing the test artifacts into a MinGit
that has busybox.exe (and using Git for Windows' SDK's Perl for now, as
the test suite requires Perl even when NO_PERL is set, go figure) by
using the `install-mingit-test-artifacts` Makefile target with the
DESTDIR variable pointing to the top-level directory of the MinGit
installation.

To facilitate running the test suite (without having `make` available,
as `make.exe` is not part of MinGit), this branch brings an experimental
patch to the `test-run-command` helper to run Git's test suite. It is
still very experimental, though: in this developer's tests it seemed
that the `poll()` emulation required for `run_parallel_processes()` to
work sometimes hiccups on Windows, causing infinite "hangs". It is also
possible that BusyBox itself has problems writing to the pipes opened by
`test-run-command` (and merging this branch will help investigate
further). Caveat emptor.

Signed-off-by: Johannes Schindelin <[email protected]>

Author: Git for Windows Build Agent

Makefile

@@ -655,6 +655,7 @@ TEST_PROGRAMS_NEED_X += test-dump-untracked-cache
 TEST_PROGRAMS_NEED_X += test-fake-ssh
 TEST_PROGRAMS_NEED_X += test-genrandom
 TEST_PROGRAMS_NEED_X += test-hashmap
+TEST_PROGRAMS_NEED_X += test-helper
 TEST_PROGRAMS_NEED_X += test-index-version
 TEST_PROGRAMS_NEED_X += test-lazy-init-name-hash
 TEST_PROGRAMS_NEED_X += test-line-buffer

compat/mingw.c

@@ -8,6 +8,7 @@
 #include "../cache.h"
 #include "win32/exit-process.h"
 #include "../config.h"
+#include "../string-list.h"
 
 #define HCAST(type, handle) ((type)(intptr_t)handle)
 
@@ -1216,6 +1217,65 @@ static char *lookup_prog(const char *dir, int dirlen, const char *cmd,
 	return NULL;
 }
 
+static char *path_lookup(const char *cmd, int exe_only);
+
+static char *is_busybox_applet(const char *cmd)
+{
+	static struct string_list applets = STRING_LIST_INIT_DUP;
+	static char *busybox_path;
+	static int busybox_path_initialized;
+
+	/* Avoid infinite loop */
+	if (!strncasecmp(cmd, "busybox", 7) &&
+	    (!cmd[7] || !strcasecmp(cmd + 7, ".exe")))
+		return NULL;
+
+	if (!busybox_path_initialized) {
+		busybox_path = path_lookup("busybox.exe", 1);
+		busybox_path_initialized = 1;
+	}
+
+	/* Assume that sh is compiled in... */
+	if (!busybox_path || !strcasecmp(cmd, "sh"))
+		return xstrdup_or_null(busybox_path);
+
+	if (!applets.nr) {
+		struct child_process cp = CHILD_PROCESS_INIT;
+		struct strbuf buf = STRBUF_INIT;
+		char *p;
+
+		argv_array_pushl(&cp.args, busybox_path, "--help", NULL);
+
+		if (capture_command(&cp, &buf, 2048)) {
+			string_list_append(&applets, "");
+			return NULL;
+		}
+
+		/* parse output */
+		p = strstr(buf.buf, "Currently defined functions:\n");
+		if (!p) {
+			warning("Could not parse output of busybox --help");
+			string_list_append(&applets, "");
+			return NULL;
+		}
+		p = strchrnul(p, '\n');
+		for (;;) {
+			size_t len;
+
+			p += strspn(p, "\n\t ,");
+			len = strcspn(p, "\n\t ,");
+			if (!len)
+				break;
+			p[len] = '\0';
+			string_list_insert(&applets, p);
+			p = p + len + 1;
+		}
+	}
+
+	return string_list_has_string(&applets, cmd) ?
+		xstrdup(busybox_path) : NULL;
+}
+
 /*
  * Determines the absolute path of cmd using the split path in path.
  * If cmd contains a slash or backslash, no lookup is performed.
@@ -1244,6 +1304,9 @@ static char *path_lookup(const char *cmd, int exe_only)
 		path = sep + 1;
 	}
 
+	if (!prog && !isexe)
+		prog = is_busybox_applet(cmd);
+
 	return prog;
 }
 
@@ -1557,8 +1620,8 @@ static int is_msys2_sh(const char *cmd)
 }
 
 static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaenv,
-			      const char *dir,
-			      int prepend_cmd, int fhin, int fhout, int fherr)
+			      const char *dir, const char *prepend_cmd,
+			      int fhin, int fhout, int fherr)
 {
 	static int atexit_handler_initialized;
 	STARTUPINFOW si;
@@ -1627,9 +1690,9 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen
 	/* concatenate argv, quoting args as we go */
 	strbuf_init(&args, 0);
 	if (prepend_cmd) {
-		char *quoted = (char *)quote_arg(cmd);
+		char *quoted = (char *)quote_arg(prepend_cmd);
 		strbuf_addstr(&args, quoted);
-		if (quoted != cmd)
+		if (quoted != prepend_cmd)
 			free(quoted);
 	}
 	for (; *argv; argv++) {
@@ -1706,7 +1769,8 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen
 	return (pid_t)pi.dwProcessId;
 }
 
-static pid_t mingw_spawnv(const char *cmd, const char **argv, int prepend_cmd)
+static pid_t mingw_spawnv(const char *cmd, const char **argv,
+			  const char *prepend_cmd)
 {
 	return mingw_spawnve_fd(cmd, argv, NULL, NULL, prepend_cmd, 0, 1, 2);
 }
@@ -1734,14 +1798,14 @@ pid_t mingw_spawnvpe(const char *cmd, const char **argv, char **deltaenv,
 				pid = -1;
 			}
 			else {
-				pid = mingw_spawnve_fd(iprog, argv, deltaenv, dir, 1,
+				pid = mingw_spawnve_fd(iprog, argv, deltaenv, dir, interpr,
 						       fhin, fhout, fherr);
 				free(iprog);
 			}
 			argv[0] = argv0;
 		}
 		else
-			pid = mingw_spawnve_fd(prog, argv, deltaenv, dir, 0,
+			pid = mingw_spawnve_fd(prog, argv, deltaenv, dir, NULL,
 					       fhin, fhout, fherr);
 		free(prog);
 	}
@@ -1767,7 +1831,7 @@ static int try_shell_exec(const char *cmd, char *const *argv)
 		ALLOC_ARRAY(argv2, argc + 1);
 		argv2[0] = (char *)cmd;	/* full path to the script file */
 		memcpy(&argv2[1], &argv[1], sizeof(*argv) * argc);
-		pid = mingw_spawnv(prog, argv2, 1);
+		pid = mingw_spawnv(prog, argv2, interpr);
 		if (pid >= 0) {
 			int status;
 			if (waitpid(pid, &status, 0) < 0)
@@ -1787,7 +1851,7 @@ int mingw_execv(const char *cmd, char *const *argv)
 	if (!try_shell_exec(cmd, argv)) {
 		int pid, status;
 
-		pid = mingw_spawnv(cmd, (const char **)argv, 0);
+		pid = mingw_spawnv(cmd, (const char **)argv, NULL);
 		if (pid < 0)
 			return -1;
 		if (waitpid(pid, &status, 0) < 0)

config.mak.uname

@@ -688,6 +688,65 @@ else
 		NO_CURL = YesPlease
 	endif
 endif
+ifeq (i686,$(uname_M))
+	MINGW_PREFIX := mingw32
+endif
+ifeq (x86_64,$(uname_M))
+	MINGW_PREFIX := mingw64
+endif
+
+	DESTDIR_WINDOWS = $(shell cygpath -aw '$(DESTDIR_SQ)')
+	DESTDIR_MIXED = $(shell cygpath -am '$(DESTDIR_SQ)')
+install-mingit-test-artifacts:
+	install -m755 -d '$(DESTDIR_SQ)/usr/bin'
+	printf '%s\n%s\n' >'$(DESTDIR_SQ)/usr/bin/perl' \
+		"#!/mingw64/bin/busybox sh" \
+		"exec \"$(shell cygpath -am /usr/bin/perl.exe)\" \"\$$@\""
+
+	install -m755 -d '$(DESTDIR_SQ)'
+	printf '%s%s\n%s\n%s\n%s\n%s\n' >'$(DESTDIR_SQ)/init.bat' \
+		"PATH=$(DESTDIR_WINDOWS)\\$(MINGW_PREFIX)\\bin;" \
+		"C:\\WINDOWS;C:\\WINDOWS\\system32" \
+		"@set GIT_TEST_INSTALLED=$(DESTDIR_MIXED)/$(MINGW_PREFIX)/bin" \
+		"@`echo "$(DESTDIR_WINDOWS)" | sed 's/:.*/:/'`" \
+		"@cd `echo "$(DESTDIR_WINDOWS)" | sed 's/^.://'`\\test-git\\t" \
+		"@echo Now, run 'helper\\test-run-command testsuite'"
+
+	install -m755 -d '$(DESTDIR_SQ)/test-git'
+	sed 's/^\(NO_PERL\|NO_PYTHON\)=.*/\1=YesPlease/' \
+		<GIT-BUILD-OPTIONS >'$(DESTDIR_SQ)/test-git/GIT-BUILD-OPTIONS'
+
+	install -m755 -d '$(DESTDIR_SQ)/test-git/t/helper'
+	install -m755 $(TEST_PROGRAMS) '$(DESTDIR_SQ)/test-git/t/helper'
+	(cd t && $(TAR) cf - t[0-9][0-9][0-9][0-9] diff-lib) | \
+	(cd '$(DESTDIR_SQ)/test-git/t' && $(TAR) xf -)
+	install -m755 t/t556x_common t/*.sh '$(DESTDIR_SQ)/test-git/t'
+
+	install -m755 -d '$(DESTDIR_SQ)/test-git/templates'
+	(cd templates && $(TAR) cf - blt) | \
+	(cd '$(DESTDIR_SQ)/test-git/templates' && $(TAR) xf -)
+
+	# po/build/locale for t0200
+	install -m755 -d '$(DESTDIR_SQ)/test-git/po/build/locale'
+	(cd po/build/locale && $(TAR) cf - .) | \
+	(cd '$(DESTDIR_SQ)/test-git/po/build/locale' && $(TAR) xf -)
+
+	# git-daemon.exe for t5802, git-http-backend.exe for t5560
+	install -m755 -d '$(DESTDIR_SQ)/$(MINGW_PREFIX)/bin'
+	install -m755 git-daemon.exe git-http-backend.exe \
+		'$(DESTDIR_SQ)/$(MINGW_PREFIX)/bin'
+
+	# git-remote-testgit for t5801
+	install -m755 -d '$(DESTDIR_SQ)/$(MINGW_PREFIX)/libexec/git-core'
+	install -m755 git-remote-testgit \
+		'$(DESTDIR_SQ)/$(MINGW_PREFIX)/libexec/git-core'
+
+	# git-upload-archive (dashed) for t5000
+	install -m755 git-upload-archive.exe '$(DESTDIR_SQ)/$(MINGW_PREFIX)/bin'
+
+	# git-difftool--helper for t7800
+	install -m755 git-difftool--helper \
+		'$(DESTDIR_SQ)/$(MINGW_PREFIX)/libexec/git-core'
 endif
 ifeq ($(uname_S),QNX)
 	COMPAT_CFLAGS += -DSA_RESTART=0

git-sh-setup.sh

@@ -332,17 +332,30 @@ create_virtual_base() {
 # Platform specific tweaks to work around some commands
 case $(uname -s) in
 *MINGW*)
-	# Windows has its own (incompatible) sort and find
-	sort () {
-		/usr/bin/sort "$@"
-	}
-	find () {
-		/usr/bin/find "$@"
-	}
-	# git sees Windows-style pwd
-	pwd () {
-		builtin pwd -W
-	}
+	if test -x /usr/bin/sort
+	then
+		# Windows has its own (incompatible) sort; override
+		sort () {
+			/usr/bin/sort "$@"
+		}
+	fi
+	if test -x /usr/bin/find
+	then
+		# Windows has its own (incompatible) find; override
+		find () {
+			/usr/bin/find "$@"
+		}
+	fi
+	# On Windows, Git wants Windows paths. But /usr/bin/pwd spits out
+	# Unix-style paths. At least in Bash, we have a builtin pwd that
+	# understands the -W option to force "mixed" paths, i.e. with drive
+	# prefix but still with forward slashes. Let's use that, if available.
+	if type builtin >/dev/null 2>&1
+	then
+		pwd () {
+			builtin pwd -W
+		}
+	fi
 	is_absolute_path () {
 		case "$1" in
 		[/\\]* | [A-Za-z]:*)

t/.gitattributes

@@ -1,5 +1,6 @@
 t[0-9][0-9][0-9][0-9]/* -whitespace
 /diff-lib/* eol=lf
+/diff-lib/*.png binary
 /t0110/url-* binary
 /t3900/*.txt eol=lf
 /t3901/*.txt eol=lf

t/test-binary-1.png renamed to t/diff-lib/test-binary-1.png

@@ -12,6 +12,7 @@
 /test-scrap-cache-tree
 /test-genrandom
 /test-hashmap
+/test-helper
 /test-index-version
 /test-lazy-init-name-hash
 /test-line-buffer

t/test-binary-2.png renamed to t/diff-lib/test-binary-2.png

@@ -0,0 +1,113 @@
+#include "git-compat-util.h"
+#include "strbuf.h"
+#include "gettext.h"
+#include "parse-options.h"
+#include "utf8.h"
+
+static const char * const test_helper_usage[] = {
+	N_("test-helper [<options>]"),
+	NULL
+};
+
+static int cmp(int argc, const char **argv)
+{
+	FILE *f0, *f1;
+	struct strbuf b0 = STRBUF_INIT, b1 = STRBUF_INIT;
+
+	if (argc != 3)
+		die("Require exactly 2 arguments, got %d", argc);
+
+	if (!(f0 = !strcmp(argv[1], "-") ? stdin : fopen(argv[1], "r")))
+		return error_errno("could not open '%s'", argv[1]);
+	if (!(f1 = !strcmp(argv[2], "-") ? stdin : fopen(argv[2], "r"))) {
+		fclose(f0);
+		return error_errno("could not open '%s'", argv[2]);
+	}
+
+	for (;;) {
+		int r0 = strbuf_getline(&b0, f0);
+		int r1 = strbuf_getline(&b1, f1);
+
+		if (r0 == EOF) {
+			fclose(f0);
+			fclose(f1);
+			strbuf_release(&b0);
+			strbuf_release(&b1);
+			if (r1 == EOF)
+				return 0;
+			return 1;
+		}
+		if (r1 == EOF || strbuf_cmp(&b0, &b1)) {
+			fclose(f0);
+			fclose(f1);
+			strbuf_release(&b0);
+			strbuf_release(&b1);
+			return 1;
+		}
+	}
+}
+
+static int iconv_(int argc, const char **argv)
+{
+	struct strbuf buf = STRBUF_INIT;
+	char *from = NULL, *to = NULL, *p;
+	int len, ret = 0;
+	const char * const iconv_usage[] = {
+		N_("test-helper --iconv [<options>]"),
+		NULL
+	};
+	struct option options[] = {
+		OPT_STRING('f', "from-code", &from, "encoding", "from"),
+		OPT_STRING('t', "to-code", &to, "encoding", "to"),
+		OPT_END()
+	};
+
+	argc = parse_options(argc, argv, NULL, options,
+			iconv_usage, 0);
+
+	if (argc > 1 || !from || !to)
+		usage_with_options(iconv_usage, options);
+
+	if (!argc) {
+		if (strbuf_read(&buf, 0, 2048) < 0)
+			die_errno("Could not read from stdin");
+	} else if (strbuf_read_file(&buf, argv[0], 2048) < 0)
+		die_errno("Could not read from '%s'", argv[0]);
+
+	p = reencode_string_len(buf.buf, buf.len, to, from, &len);
+	if (!p)
+		die_errno("Could not reencode");
+	if (write(1, p, len) < 0)
+		ret = !!error_errno("Could not write %d bytes", len);
+
+	strbuf_release(&buf);
+	free(p);
+
+	return ret;
+}
+
+int cmd_main(int argc, const char **argv)
+{
+	enum mode {
+		CMP = 1, ICONV
+	} command = 0;
+	struct option options[] = {
+		OPT_CMDMODE(0, "cmp", &command,
+			N_("compare files (ignoring LF vs CR/LF)"), CMP),
+		OPT_CMDMODE(0, "iconv", &command,
+			N_("act as drop-in replacement for `iconv`"), ICONV),
+		OPT_END()
+	};
+
+	argc = parse_options(argc, argv, NULL, options,
+			test_helper_usage,
+			PARSE_OPT_KEEP_ARGV0 | PARSE_OPT_KEEP_UNKNOWN);
+
+	if (command == CMP)
+		return !!cmp(argc, argv);
+	if (command == ICONV)
+		return !!iconv_(argc, argv);
+
+	die("unhandled mode");
+}
+