Revert "Merge branch 'mingw-isatty-fixup'"

Prepare to merge the most recent iteration of the mingw-isatty patches.

This reverts commit 66d27de, reversing
changes made to cf8df3d.

Signed-off-by: Johannes Schindelin <[email protected]>

Author: dscho

compat/msvc.h

@@ -31,6 +31,9 @@ static __inline int strcasecmp (const char *s1, const char *s2)
 #ifdef _MSC_VER
 #define ftello _ftelli64
 
+#define isatty msc_isatty
+int msc_isatty(int);
+
 typedef int sigset_t;
 /* open for reading, writing, or both (not in fcntl.h) */
 #define O_ACCMODE     (_O_RDONLY | _O_WRONLY | _O_RDWR)

compat/winansi.c

@@ -8,10 +8,24 @@
 #include <winreg.h>
 #include "win32.h"
 
+#if defined(_MSC_VER)
+
 static int fd_is_interactive[3] = { 0, 0, 0 };
-#define FD_CONSOLE 0x1
-#define FD_SWAPPED 0x2
-#define FD_MSYS    0x4
+#define MY_INTERACTIVE_CONSOLE 0x1
+#define MY_INTERACTIVE_SWAPPED 0x2
+#define MY_INTERACTIVE_MSYS    0x4
+
+/* Accumulate what we know about the inherited console descriptors. */
+static void set_interactive(int fd, int bit)
+{
+	if (fd >=0 && fd <= 2)
+		fd_is_interactive[fd] |= bit;
+}
+
+#endif
+
+/* In this file, we actually want to use Windows' own isatty(). */
+#undef isatty
 
 /*
  ANSI codes used by git: m, K
@@ -82,7 +96,6 @@ static void warn_if_raster_font(void)
 static int is_console(int fd)
 {
 	CONSOLE_SCREEN_BUFFER_INFO sbi;
-	DWORD mode;
 	HANDLE hcon;
 
 	static int initialized = 0;
@@ -97,14 +110,12 @@ static int is_console(int fd)
 		return 0;
 
 	/* check if its a handle to a console output screen buffer */
-	if (!fd) {
-		if (!GetConsoleMode(hcon, &mode))
-			return 0;
-	} else if (!GetConsoleScreenBufferInfo(hcon, &sbi))
+	if (!GetConsoleScreenBufferInfo(hcon, &sbi))
 		return 0;
 
-	if (fd >= 0 && fd <= 2)
-		fd_is_interactive[fd] |= FD_CONSOLE;
+#if defined(_MSC_VER)
+	set_interactive(fd, MY_INTERACTIVE_CONSOLE);
+#endif
 
 	/* initialize attributes */
 	if (!initialized) {
@@ -467,52 +478,131 @@ static HANDLE duplicate_handle(HANDLE hnd)
 	return hresult;
 }
 
+#if defined(_MSC_VER)
 static HANDLE swap_osfhnd(int fd, HANDLE new_handle)
 {
+	DWORD key_std = ((fd == 1) ? STD_OUTPUT_HANDLE : STD_ERROR_HANDLE);
+
 	/*
 	 * Create a copy of the original handle associated with fd
 	 * because the original will get closed when we dup2().
 	 */
-	HANDLE handle = (HANDLE)_get_osfhandle(fd);
-	HANDLE duplicate = duplicate_handle(handle);
+	HANDLE h_original = (HANDLE)_get_osfhandle(fd);
+	HANDLE h_copy_original = duplicate_handle(h_original);
 
 	/* Create a temp fd associated with the already open "new_handle". */
-	int new_fd = _open_osfhandle((intptr_t)new_handle, O_BINARY);
+	int fd_temp = _open_osfhandle((intptr_t)new_handle, O_BINARY);
 
 	assert((fd == 1) || (fd == 2));
 
 	/*
 	 * Use stock dup2() to re-bind fd to the new handle.  Note that
 	 * this will implicitly close(1) and close both fd=1 and the
 	 * originally associated handle.  It will open a new fd=1 and
-	 * call DuplicateHandle() on the handle associated with new_fd.
+	 * call DuplicateHandle() on the handle associated with fd_temp.
 	 * It is because of this implicit close() that we created the
 	 * copy of the original.
 	 *
 	 * Note that the OS can recycle HANDLE (numbers) just like it
 	 * recycles fd (numbers), so we must update the cached value
 	 * of "console".  You can use GetFileType() to see that
-	 * handle and _get_osfhandle(fd) may have the same number
+	 * h_original and _get_osfhandle(fd) may have the same number
 	 * value, but they refer to different actual files now.
 	 *
 	 * Note that dup2() when given target := {0,1,2} will also
 	 * call SetStdHandle(), so we don't need to worry about that.
 	 */
-	dup2(new_fd, fd);
-	if (console == handle)
-		console = duplicate;
-	handle = INVALID_HANDLE_VALUE;
+	dup2(fd_temp, fd);
+	if (console == h_original)
+		console = h_copy_original;
+	h_original = INVALID_HANDLE_VALUE;
 
 	/* Close the temp fd.  This explicitly closes "new_handle"
 	 * (because it has been associated with it).
 	 */
-	close(new_fd);
+	close(fd_temp);
+
+	fd_is_interactive[fd] |= MY_INTERACTIVE_SWAPPED;
+
+	return h_copy_original;
+}
+
+#else
+
+/*
+ * Make MSVCRT's internal file descriptor control structure accessible
+ * so that we can tweak OS handles and flags directly (we need MSVCRT
+ * to treat our pipe handle as if it were a console).
+ *
+ * We assume that the ioinfo structure (exposed by MSVCRT.dll via
+ * __pioinfo) starts with the OS handle and the flags. The exact size
+ * varies between MSVCRT versions, so we try different sizes until
+ * toggling the FDEV bit of _pioinfo(1)->osflags is reflected in
+ * isatty(1).
+ */
+typedef struct {
+	HANDLE osfhnd;
+	char osflags;
+} ioinfo;
+
+extern __declspec(dllimport) ioinfo *__pioinfo[];
+
+static size_t sizeof_ioinfo = 0;
+
+#define IOINFO_L2E 5
+#define IOINFO_ARRAY_ELTS (1 << IOINFO_L2E)
+
+#define FPIPE 0x08
+#define FDEV  0x40
+
+static inline ioinfo* _pioinfo(int fd)
+{
+	return (ioinfo*)((char*)__pioinfo[fd >> IOINFO_L2E] +
+			(fd & (IOINFO_ARRAY_ELTS - 1)) * sizeof_ioinfo);
+}
 
-	fd_is_interactive[fd] |= FD_SWAPPED;
+static int init_sizeof_ioinfo(void)
+{
+	int istty, wastty;
+	/* don't init twice */
+	if (sizeof_ioinfo)
+		return sizeof_ioinfo >= 256;
+
+	sizeof_ioinfo = sizeof(ioinfo);
+	wastty = isatty(1);
+	while (sizeof_ioinfo < 256) {
+		/* toggle FDEV flag, check isatty, then toggle back */
+		_pioinfo(1)->osflags ^= FDEV;
+		istty = isatty(1);
+		_pioinfo(1)->osflags ^= FDEV;
+		/* return if we found the correct size */
+		if (istty != wastty)
+			return 0;
+		sizeof_ioinfo += sizeof(void*);
+	}
+	error("Tweaking file descriptors doesn't work with this MSVCRT.dll");
+	return 1;
+}
 
-	return duplicate;
+static HANDLE swap_osfhnd(int fd, HANDLE new_handle)
+{
+	ioinfo *pioinfo;
+	HANDLE old_handle;
+
+	/* init ioinfo size if we haven't done so */
+	if (init_sizeof_ioinfo())
+		return INVALID_HANDLE_VALUE;
+
+	/* get ioinfo pointer and change the handles */
+	pioinfo = _pioinfo(fd);
+	old_handle = pioinfo->osfhnd;
+	pioinfo->osfhnd = new_handle;
+	return old_handle;
 }
 
+#endif
+
+
 #ifdef DETECT_MSYS_TTY
 
 #include <winternl.h>
@@ -558,25 +648,49 @@ static void detect_msys_tty(int fd)
 			!wcsstr(name, L"-pty"))
 		return;
 
-	fd_is_interactive[fd] |= FD_MSYS;
+#if defined(_MSC_VER)
+	fd_is_interactive[fd] |= MY_INTERACTIVE_MSYS;
+#else
+	/* init ioinfo size if we haven't done so */
+	if (init_sizeof_ioinfo())
+		return;
+
+	/* set FDEV flag, reset FPIPE flag */
+	_pioinfo(fd)->osflags &= ~FPIPE;
+	_pioinfo(fd)->osflags |= FDEV;
+#endif
 }
 
 #endif
 
-/*
- * Wrapper for isatty().  Most calls in the main git code
- * call isatty(1 or 2) to see if the instance is interactive
- * and should: be colored, show progress, paginate output.
- * We lie and give results for what the descriptor WAS at
- * startup (and ignore any pipe redirection we internally
- * do).
- */
-#undef isatty
 int winansi_isatty(int fd)
 {
-	if (fd >= 0 && fd <= 2)
-		return fd_is_interactive[fd] != 0;
-	return isatty(fd);
+	int res = isatty(fd);
+
+	if (res) {
+		/*
+		 * Make sure that /dev/null is not fooling Git into believing
+		 * that we are connected to a terminal, as "_isatty() returns a
+		 * nonzero value if the descriptor is associated with a
+		 * character device."; for more information, see
+		 *
+		 * https://msdn.microsoft.com/en-us/library/f4s0ddew.aspx
+		 */
+		HANDLE handle = (HANDLE)_get_osfhandle(fd);
+		if (fd == STDIN_FILENO) {
+			DWORD dummy;
+
+			if (!GetConsoleMode(handle, &dummy))
+				res = 0;
+		} else if (fd == STDOUT_FILENO || fd == STDERR_FILENO) {
+			CONSOLE_SCREEN_BUFFER_INFO dummy;
+
+			if (!GetConsoleScreenBufferInfo(handle, &dummy))
+				res = 0;
+		}
+	}
+
+	return res;
 }
 
 void winansi_init(void)
@@ -588,8 +702,10 @@ void winansi_init(void)
 	con1 = is_console(1);
 	con2 = is_console(2);
 
+#if defined(_MSC_VER)
 	/* Also compute console bit for fd 0 even though we don't need the result here. */
 	is_console(0);
+#endif
 
 	if (!con1 && !con2) {
 #ifdef DETECT_MSYS_TTY
@@ -634,10 +750,32 @@ void winansi_init(void)
  */
 HANDLE winansi_get_osfhandle(int fd)
 {
-	if (fd == 1 && (fd_is_interactive[1] & FD_SWAPPED))
-		return hconsole1;
-	if (fd == 2 && (fd_is_interactive[2] & FD_SWAPPED))
-		return hconsole2;
+	HANDLE hnd = (HANDLE) _get_osfhandle(fd);
+	if (isatty(fd) && GetFileType(hnd) == FILE_TYPE_PIPE) {
+		if (fd == 1 && hconsole1)
+			return hconsole1;
+		else if (fd == 2 && hconsole2)
+			return hconsole2;
+	}
+	return hnd;
+}
+
+#ifdef _MSC_VER
 
-	return (HANDLE)_get_osfhandle(fd);
+/* Wrapper for isatty().  Most calls in the main git code
+ * call isatty(1 or 2) to see if the instance is interactive
+ * and should: be colored, show progress, paginate output.
+ * We lie and give results for what the descriptor WAS at
+ * startup (and ignore any pipe redirection we internally
+ * do).
+ */
+#undef isatty
+int msc_isatty(fd)
+{
+	if (fd >=0 && fd <= 2)
+		return fd_is_interactive[fd] != 0;
+	else
+		return isatty(fd);
 }
+
+#endif