Merge pull request #102 from kb/msys-tty

Better MSys PTY detection

Signed-off-by: Johannes Schindelin <[email protected]>

Author: dscho

compat/mingw.c

@@ -6,8 +6,6 @@
 #include "../run-command.h"
 #include "../cache.h"
 
-#undef isatty
-
 #define HCAST(type, handle) ((type)(intptr_t)handle)
 
 static const int delay[] = { 0, 1, 10, 20, 40 };
@@ -2380,38 +2378,3 @@ void mingw_startup()
 	/* init length of current directory for handle_long_path */
 	current_directory_len = GetCurrentDirectoryW(0, NULL);
 }
-
-int mingw_isatty(int fd) {
-	static DWORD id[] = {
-		STD_INPUT_HANDLE,
-		STD_OUTPUT_HANDLE,
-		STD_ERROR_HANDLE
-	};
-	static unsigned initialized;
-	static int is_tty[ARRAY_SIZE(id)];
-
-	if (fd < 0 || fd >= ARRAY_SIZE(is_tty))
-		return isatty(fd);
-
-	if (isatty(fd))
-		return 1;
-
-	if (!initialized) {
-		const char *env = getenv("MSYS_TTY_HANDLES");
-
-		if (env) {
-			int i;
-			char buffer[64];
-
-			for (i = 0; i < ARRAY_SIZE(is_tty); i++) {
-				sprintf(buffer, " %" PRIuMAX " ",
-					HCAST(uintmax_t, GetStdHandle(id[i])));
-				is_tty[i] = !!strstr(env, buffer);
-			}
-		}
-
-		initialized = 1;
-	}
-
-	return is_tty[fd];
-}

compat/mingw.h

@@ -367,9 +367,6 @@ sig_handler_t mingw_signal(int sig, sig_handler_t handler);
 int mingw_raise(int sig);
 #define raise mingw_raise
 
-int mingw_isatty(int fd);
-#define isatty mingw_isatty
-
 /*
  * ANSI emulation wrappers
  */

compat/winansi.c

@@ -7,8 +7,6 @@
 #include <wingdi.h>
 #include <winreg.h>
 
-#undef isatty
-
 /*
  ANSI codes used by git: m, K
 
@@ -485,6 +483,7 @@ static size_t sizeof_ioinfo = 0;
 #define IOINFO_L2E 5
 #define IOINFO_ARRAY_ELTS (1 << IOINFO_L2E)
 
+#define FPIPE 0x08
 #define FDEV  0x40
 
 static inline ioinfo* _pioinfo(int fd)
@@ -532,6 +531,45 @@ static HANDLE swap_osfhnd(int fd, HANDLE new_handle)
 	return old_handle;
 }
 
+#ifdef DETECT_MSYS_TTY
+
+#include <winternl.h>
+#include <ntstatus.h>
+
+static void detect_msys_tty(int fd)
+{
+	ULONG result;
+	BYTE buffer[1024];
+	POBJECT_NAME_INFORMATION nameinfo = (POBJECT_NAME_INFORMATION) buffer;
+	PWSTR name;
+
+	/* check if fd is a pipe */
+	HANDLE h = (HANDLE) _get_osfhandle(fd);
+	if (GetFileType(h) != FILE_TYPE_PIPE)
+		return;
+
+	/* get pipe name */
+	if (!NT_SUCCESS(NtQueryObject(h, ObjectNameInformation,
+			buffer, sizeof(buffer) - 2, &result)))
+		return;
+	name = nameinfo->Name.Buffer;
+	name[nameinfo->Name.Length] = 0;
+
+	/* check if this could be a msys pty pipe ('msys-XXXX-ptyN-XX') */
+	if (!wcsstr(name, L"msys-") || !wcsstr(name, L"-pty"))
+		return;
+
+	/* init ioinfo size if we haven't done so */
+	if (init_sizeof_ioinfo())
+		return;
+
+	/* set FDEV flag, reset FPIPE flag */
+	_pioinfo(fd)->osflags &= ~FPIPE;
+	_pioinfo(fd)->osflags |= FDEV;
+}
+
+#endif
+
 void winansi_init(void)
 {
 	int con1, con2;
@@ -540,8 +578,15 @@ void winansi_init(void)
 	/* check if either stdout or stderr is a console output screen buffer */
 	con1 = is_console(1);
 	con2 = is_console(2);
-	if (!con1 && !con2)
+	if (!con1 && !con2) {
+#ifdef DETECT_MSYS_TTY
+		/* check if stdin / stdout / stderr are msys pty pipes */
+		detect_msys_tty(0);
+		detect_msys_tty(1);
+		detect_msys_tty(2);
+#endif
 		return;
+	}
 
 	/* create a named pipe to communicate with the console thread */
 	sprintf(name, "\\\\.\\pipe\\winansi%lu", GetCurrentProcessId());
@@ -577,8 +622,11 @@ void winansi_init(void)
 HANDLE winansi_get_osfhandle(int fd)
 {
 	HANDLE hnd = (HANDLE) _get_osfhandle(fd);
-	if ((fd == 1 || fd == 2) && isatty(fd)
-	    && GetFileType(hnd) == FILE_TYPE_PIPE)
-		return (fd == 1) ? hconsole1 : hconsole2;
+	if (isatty(fd) && GetFileType(hnd) == FILE_TYPE_PIPE) {
+		if (fd == 1 && hconsole1)
+			return hconsole1;
+		else if (fd == 2 && hconsole2)
+			return hconsole2;
+	}
 	return hnd;
 }

config.mak.uname

@@ -548,7 +548,8 @@ else
 			BASIC_LDFLAGS += -Wl,--large-address-aware
 		endif
 		CC = gcc
-		COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0
+		COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0 -DDETECT_MSYS_TTY
+		EXTLIBS += -lntdll
 		INSTALL = /bin/install
 		NO_R_TO_GCC_LINKER = YesPlease
 		INTERNAL_QSORT = YesPlease