mingw: Work around MSVCRT's isatty() not knowing about MSys2

MSys2 has a slightly different notion of what constitutes a tty than
the Microsoft C runtime. The former knows whether stdin/stdout/stderr
was redirected or not, while the latter looks for a Win32 Console.

In particular when we want to know whether to spawn a pager or not, we
would rather want to know what MSys2 thinks.

We are about to introduce a change to the msys2-runtime that sets an
environment variable MSYS_TTY_HANDLES to a list of Win32 handles that
correspond to stdin/stdout/stderr, respectively, *but skips* handles that
MSys2 does not think are terminals.

This commit handles that input to augment the isatty() function to return
1 also when MSYS_TTY_HANDLES contains the corresponding handle.

The only time when Git needs to know whether a Console is attached or not
is when winansi.c is asked to Do Its Thing, therefore we refrain from
overriding isatty there.

Note: this was an issue with MSys1-based Git for Windows, too, hidden by
the fact that Git for Windows used `cmd.exe` as a terminal -- which is
backed by a real Win32 Console. Had MSys1 used, say, rxvt as its default
terminal, the symptom would have been that "git log" does not spawn a
pager by default but instead outputs the entire history (without color
coding, too). In MSys2, the default terminal is mintty, therefore we
finally could not avoid to address the issue.

Signed-off-by: Johannes Schindelin <[email protected]>

Author: dscho

compat/mingw.c

@@ -6,6 +6,8 @@
 #include "../run-command.h"
 #include "../cache.h"
 
+#undef isatty
+
 static const int delay[] = { 0, 1, 10, 20, 40 };
 
 int err_win_to_posix(DWORD winerr)
@@ -2161,3 +2163,38 @@ void mingw_startup()
 	/* initialize Unicode console */
 	winansi_init();
 }
+
+int mingw_isatty(int fd) {
+	static DWORD id[] = {
+		STD_INPUT_HANDLE,
+		STD_OUTPUT_HANDLE,
+		STD_ERROR_HANDLE
+	};
+	static unsigned initialized;
+	static int is_tty[ARRAY_SIZE(id)];
+
+	if (fd < 0 || fd >= ARRAY_SIZE(is_tty))
+		return isatty(fd);
+
+	if (isatty(fd))
+		return 1;
+
+	if (!initialized) {
+		const char *env = getenv("MSYS_TTY_HANDLES");
+
+		if (env) {
+			int i;
+			char buffer[64];
+
+			for (i = 0; i < ARRAY_SIZE(is_tty); i++) {
+				sprintf(buffer, " %ld ", (unsigned long)
+					GetStdHandle(id[i]));
+				is_tty[i] = !!strstr(env, buffer);
+			}
+		}
+
+		initialized = 1;
+	}
+
+	return is_tty[fd];
+}

compat/mingw.h

@@ -351,6 +351,9 @@ sig_handler_t mingw_signal(int sig, sig_handler_t handler);
 int mingw_raise(int sig);
 #define raise mingw_raise
 
+int mingw_isatty(int fd);
+#define isatty mingw_isatty
+
 /*
  * ANSI emulation wrappers
  */

compat/winansi.c

@@ -7,6 +7,8 @@
 #include <wingdi.h>
 #include <winreg.h>
 
+#undef isatty
+
 /*
  ANSI codes used by git: m, K