Merge branch 'mingw-isatty-fixup-v3'

This is an evil merge: it changes more than the merged commits, as the
merged branch replaces part of the MSVC patches.

This mess will need to be cleaned up in the next merging rebase, by
moving the mingw-isatty-fixup patches in front of the MSVC patches.

Signed-off-by: Johannes Schindelin <[email protected]>

Author: dscho

compat/mingw.h

@@ -472,6 +472,9 @@ int mingw_raise(int sig);
  * ANSI emulation wrappers
  */
 
+int winansi_isatty(int fd);
+#define isatty winansi_isatty
+
 void winansi_init(void);
 HANDLE winansi_get_osfhandle(int fd);
 

compat/msvc.h

@@ -31,9 +31,6 @@ static __inline int strcasecmp (const char *s1, const char *s2)
 #ifdef _MSC_VER
 #define ftello _ftelli64
 
-#define isatty msc_isatty
-int msc_isatty(int);
-
 typedef int sigset_t;
 /* open for reading, writing, or both (not in fcntl.h) */
 #define O_ACCMODE     (_O_RDONLY | _O_WRONLY | _O_RDWR)

compat/winansi.c

@@ -8,21 +8,10 @@
 #include <winreg.h>
 #include "win32.h"
 
-#if defined(_MSC_VER)
-
 static int fd_is_interactive[3] = { 0, 0, 0 };
-#define MY_INTERACTIVE_CONSOLE 0x1
-#define MY_INTERACTIVE_SWAPPED 0x2
-#define MY_INTERACTIVE_MSYS    0x4
-
-/* Accumulate what we know about the inherited console descriptors. */
-static void set_interactive(int fd, int bit)
-{
-	if (fd >=0 && fd <= 2)
-		fd_is_interactive[fd] |= bit;
-}
-
-#endif
+#define FD_CONSOLE 0x1
+#define FD_SWAPPED 0x2
+#define FD_MSYS    0x4
 
 /*
  ANSI codes used by git: m, K
@@ -93,6 +82,7 @@ static void warn_if_raster_font(void)
 static int is_console(int fd)
 {
 	CONSOLE_SCREEN_BUFFER_INFO sbi;
+	DWORD mode;
 	HANDLE hcon;
 
 	static int initialized = 0;
@@ -107,12 +97,14 @@ static int is_console(int fd)
 		return 0;
 
 	/* check if its a handle to a console output screen buffer */
-	if (!GetConsoleScreenBufferInfo(hcon, &sbi))
+	if (!fd) {
+		if (!GetConsoleMode(hcon, &mode))
+			return 0;
+	} else if (!GetConsoleScreenBufferInfo(hcon, &sbi))
 		return 0;
 
-#if defined(_MSC_VER)
-	set_interactive(fd, MY_INTERACTIVE_CONSOLE);
-#endif
+	if (fd >= 0 && fd <= 2)
+		fd_is_interactive[fd] |= FD_CONSOLE;
 
 	/* initialize attributes */
 	if (!initialized) {
@@ -475,131 +467,49 @@ static HANDLE duplicate_handle(HANDLE hnd)
 	return hresult;
 }
 
-#if defined(_MSC_VER)
 static HANDLE swap_osfhnd(int fd, HANDLE new_handle)
 {
-	DWORD key_std = ((fd == 1) ? STD_OUTPUT_HANDLE : STD_ERROR_HANDLE);
-
 	/*
 	 * Create a copy of the original handle associated with fd
 	 * because the original will get closed when we dup2().
 	 */
-	HANDLE h_original = (HANDLE)_get_osfhandle(fd);
-	HANDLE h_copy_original = duplicate_handle(h_original);
+	HANDLE handle = (HANDLE)_get_osfhandle(fd);
+	HANDLE duplicate = duplicate_handle(handle);
 
 	/* Create a temp fd associated with the already open "new_handle". */
-	int fd_temp = _open_osfhandle((intptr_t)new_handle, O_BINARY);
+	int new_fd = _open_osfhandle((intptr_t)new_handle, O_BINARY);
 
 	assert((fd == 1) || (fd == 2));
 
 	/*
 	 * Use stock dup2() to re-bind fd to the new handle.  Note that
 	 * this will implicitly close(1) and close both fd=1 and the
 	 * originally associated handle.  It will open a new fd=1 and
-	 * call DuplicateHandle() on the handle associated with fd_temp.
+	 * call DuplicateHandle() on the handle associated with new_fd.
 	 * It is because of this implicit close() that we created the
 	 * copy of the original.
 	 *
-	 * Note that the OS can recycle HANDLE (numbers) just like it
-	 * recycles fd (numbers), so we must update the cached value
-	 * of "console".  You can use GetFileType() to see that
-	 * h_original and _get_osfhandle(fd) may have the same number
-	 * value, but they refer to different actual files now.
+	 * Note that we need to update the cached console handle to the
+	 * duplicated one because the dup2() call will implicitly close
+	 * the original one.
 	 *
 	 * Note that dup2() when given target := {0,1,2} will also
 	 * call SetStdHandle(), so we don't need to worry about that.
 	 */
-	dup2(fd_temp, fd);
-	if (console == h_original)
-		console = h_copy_original;
-	h_original = INVALID_HANDLE_VALUE;
+	if (console == handle)
+		console = duplicate;
+	dup2(new_fd, fd);
 
 	/* Close the temp fd.  This explicitly closes "new_handle"
 	 * (because it has been associated with it).
 	 */
-	close(fd_temp);
+	close(new_fd);
 
-	fd_is_interactive[fd] |= MY_INTERACTIVE_SWAPPED;
+	fd_is_interactive[fd] |= FD_SWAPPED;
 
-	return h_copy_original;
+	return duplicate;
 }
 
-#else
-
-/*
- * Make MSVCRT's internal file descriptor control structure accessible
- * so that we can tweak OS handles and flags directly (we need MSVCRT
- * to treat our pipe handle as if it were a console).
- *
- * We assume that the ioinfo structure (exposed by MSVCRT.dll via
- * __pioinfo) starts with the OS handle and the flags. The exact size
- * varies between MSVCRT versions, so we try different sizes until
- * toggling the FDEV bit of _pioinfo(1)->osflags is reflected in
- * isatty(1).
- */
-typedef struct {
-	HANDLE osfhnd;
-	char osflags;
-} ioinfo;
-
-extern __declspec(dllimport) ioinfo *__pioinfo[];
-
-static size_t sizeof_ioinfo = 0;
-
-#define IOINFO_L2E 5
-#define IOINFO_ARRAY_ELTS (1 << IOINFO_L2E)
-
-#define FPIPE 0x08
-#define FDEV  0x40
-
-static inline ioinfo* _pioinfo(int fd)
-{
-	return (ioinfo*)((char*)__pioinfo[fd >> IOINFO_L2E] +
-			(fd & (IOINFO_ARRAY_ELTS - 1)) * sizeof_ioinfo);
-}
-
-static int init_sizeof_ioinfo(void)
-{
-	int istty, wastty;
-	/* don't init twice */
-	if (sizeof_ioinfo)
-		return sizeof_ioinfo >= 256;
-
-	sizeof_ioinfo = sizeof(ioinfo);
-	wastty = isatty(1);
-	while (sizeof_ioinfo < 256) {
-		/* toggle FDEV flag, check isatty, then toggle back */
-		_pioinfo(1)->osflags ^= FDEV;
-		istty = isatty(1);
-		_pioinfo(1)->osflags ^= FDEV;
-		/* return if we found the correct size */
-		if (istty != wastty)
-			return 0;
-		sizeof_ioinfo += sizeof(void*);
-	}
-	error("Tweaking file descriptors doesn't work with this MSVCRT.dll");
-	return 1;
-}
-
-static HANDLE swap_osfhnd(int fd, HANDLE new_handle)
-{
-	ioinfo *pioinfo;
-	HANDLE old_handle;
-
-	/* init ioinfo size if we haven't done so */
-	if (init_sizeof_ioinfo())
-		return INVALID_HANDLE_VALUE;
-
-	/* get ioinfo pointer and change the handles */
-	pioinfo = _pioinfo(fd);
-	old_handle = pioinfo->osfhnd;
-	pioinfo->osfhnd = new_handle;
-	return old_handle;
-}
-
-#endif
-
-
 #ifdef DETECT_MSYS_TTY
 
 #include <winternl.h>
@@ -645,21 +555,27 @@ static void detect_msys_tty(int fd)
 			!wcsstr(name, L"-pty"))
 		return;
 
-#if defined(_MSC_VER)
-	fd_is_interactive[fd] |= MY_INTERACTIVE_MSYS;
-#else
-	/* init ioinfo size if we haven't done so */
-	if (init_sizeof_ioinfo())
-		return;
-
-	/* set FDEV flag, reset FPIPE flag */
-	_pioinfo(fd)->osflags &= ~FPIPE;
-	_pioinfo(fd)->osflags |= FDEV;
-#endif
+	fd_is_interactive[fd] |= FD_MSYS;
 }
 
 #endif
 
+/*
+ * Wrapper for isatty().  Most calls in the main git code
+ * call isatty(1 or 2) to see if the instance is interactive
+ * and should: be colored, show progress, paginate output.
+ * We lie and give results for what the descriptor WAS at
+ * startup (and ignore any pipe redirection we internally
+ * do).
+ */
+#undef isatty
+int winansi_isatty(int fd)
+{
+	if (fd >= 0 && fd <= 2)
+		return fd_is_interactive[fd] != 0;
+	return isatty(fd);
+}
+
 void winansi_init(void)
 {
 	int con1, con2;
@@ -669,10 +585,8 @@ void winansi_init(void)
 	con1 = is_console(1);
 	con2 = is_console(2);
 
-#if defined(_MSC_VER)
 	/* Also compute console bit for fd 0 even though we don't need the result here. */
 	is_console(0);
-#endif
 
 	if (!con1 && !con2) {
 #ifdef DETECT_MSYS_TTY
@@ -717,32 +631,10 @@ void winansi_init(void)
  */
 HANDLE winansi_get_osfhandle(int fd)
 {
-	HANDLE hnd = (HANDLE) _get_osfhandle(fd);
-	if (isatty(fd) && GetFileType(hnd) == FILE_TYPE_PIPE) {
-		if (fd == 1 && hconsole1)
-			return hconsole1;
-		else if (fd == 2 && hconsole2)
-			return hconsole2;
-	}
-	return hnd;
-}
+	if (fd == 1 && (fd_is_interactive[1] & FD_SWAPPED))
+		return hconsole1;
+	if (fd == 2 && (fd_is_interactive[2] & FD_SWAPPED))
+		return hconsole2;
 
-#ifdef _MSC_VER
-
-/* Wrapper for isatty().  Most calls in the main git code
- * call isatty(1 or 2) to see if the instance is interactive
- * and should: be colored, show progress, paginate output.
- * We lie and give results for what the descriptor WAS at
- * startup (and ignore any pipe redirection we internally
- * do).
- */
-#undef isatty
-int msc_isatty(fd)
-{
-	if (fd >=0 && fd <= 2)
-		return fd_is_interactive[fd] != 0;
-	else
-		return isatty(fd);
+	return (HANDLE)_get_osfhandle(fd);
 }
-
-#endif