[PATCH 1/3] Detect GDB is in cygwin

[Yao Qi <yao at codesourcery dot com>]

Hello,
This patch is to detect whether GDB is in cygwin by means of new added
function 'is_in_cygwin'.  In general, the detection is inspired by
Corinna's example on detecting whether GDB is running in a cygwin pty.
I extended the logic a little to handle the case we ssh to a cygwin
machine without allocating a pty.  In this patch, we'll find the file
name of stdin, if its name is prefixed by "\cygwin-", we know GDB is
running in cygwin.

The C code is simple, but getting these simple c code compiled on
various mingw compilers is not simple, due to different header files
they shipped.  I change configure.ac to first check winternl.h, if it
is not found, check other needed headers.  Then invoke AC_TRY_COMPILE
to test whether I can use function NtQueryInformationFile.

GDB is compiled with mingw32 gcc (Fedora MinGW 4.6.1-3.fc16) and
x86_64-w64-mingw32-gcc (4.8.0 20121031).

gdb:

2013-07-29  Yao Qi  <yao@codesourcery.com>
	    Corinna Vinschen  <vinschen@redhat.com>

	* configure.ac: Invoke AC_CHECK_HEADERS to check winternl.h,
	ntdef.h, winbase.h and ddk/ntddk.h.
	* config.in: Re-generated.
	* configure: Re-generated.
	* defs.h [__MINGW32__] (is_in_cygwin_p): Declare.
	* mingw-hdep.c: Inlcude wchar.h.
	[HAVE_WINTERNL_H]: Include winternl.h.
	[!HAVE_WINTERNL_H] [HAVE_WINBASE_H]: Include ntdef.h.
	[!HAVE_WINTERNL_H] [HAVE_NTDEF_H]: winbase.h.
	[!HAVE_WINTERNL_H] [HAVE_DDK_NTDDK_H]ddk/ntddk.h.
	(pNtQueryInformationFile): Declare.
	[HAVE_NTQUERYINFORMATIONFILE] (get_filename_from_handle):
	New.
	(is_in_cygwin_p): New.
---
 gdb/config.in    |   15 +++++++++
 gdb/configure    |   82 +++++++++++++++++++++++++++++++++++++++++++++++++
 gdb/configure.ac |   40 ++++++++++++++++++++++++
 gdb/defs.h       |    4 ++
 gdb/mingw-hdep.c |   90 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 231 insertions(+), 0 deletions(-)

diff --git a/gdb/config.in b/gdb/config.in
index 92c2789..4fb45b4 100644
--- a/gdb/config.in
+++ b/gdb/config.in
@@ -84,6 +84,9 @@
 /* Define to 1 if you have the <curses.h> header file. */
 #undef HAVE_CURSES_H
 
+/* Define to 1 if you have the <ddk/ntddk.h> header file. */
+#undef HAVE_DDK_NTDDK_H
+
 /* Define to 1 if you have the declaration of `ADDR_NO_RANDOMIZE', and to 0 if
    you don't. */
 #undef HAVE_DECL_ADDR_NO_RANDOMIZE
@@ -270,6 +273,12 @@
 /* Define to 1 if you have the <nlist.h> header file. */
 #undef HAVE_NLIST_H
 
+/* Define to 1 if you have the <ntdef.h> header file. */
+#undef HAVE_NTDEF_H
+
+/* Define if function NtQueryInformationFile can be used. */
+#undef HAVE_NTQUERYINFORMATIONFILE
+
 /* Define if you support the personality syscall. */
 #undef HAVE_PERSONALITY
 
@@ -572,6 +581,12 @@
 /* Define to 1 if you have the `wborder' function. */
 #undef HAVE_WBORDER
 
+/* Define to 1 if you have the <winbase.h> header file. */
+#undef HAVE_WINBASE_H
+
+/* Define to 1 if you have the <winternl.h> header file. */
+#undef HAVE_WINTERNL_H
+
 /* Define to 1 if `fork' works. */
 #undef HAVE_WORKING_FORK
 
diff --git a/gdb/configure b/gdb/configure
index 4833297..2eb3bb8 100755
--- a/gdb/configure
+++ b/gdb/configure
@@ -9030,6 +9030,38 @@ fi
 done
 
 
+# Check header winternl.h, if not found, check ntdef.h winbase.h and ddk/ntddk.h.
+case "${host}" in
+  *-*-mingw*) for ac_header in winternl.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "winternl.h" "ac_cv_header_winternl_h" "$ac_includes_default"
+if test "x$ac_cv_header_winternl_h" = x""yes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_WINTERNL_H 1
+_ACEOF
+
+else
+  for ac_header in ntdef.h winbase.h ddk/ntddk.h
+do :
+  as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh`
+ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default"
+eval as_val=\$$as_ac_Header
+   if test "x$as_val" = x""yes; then :
+  cat >>confdefs.h <<_ACEOF
+#define `$as_echo "HAVE_$ac_header" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+
+done
+
+fi
+
+done
+
+		;;
+esac
+
 # ------------------------- #
 # Checks for declarations.  #
 # ------------------------- #
@@ -11962,6 +11994,56 @@ fi
 $as_echo "$found" >&6; }
 
 
+
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#include <windows.h>
+int
+main ()
+{
+
+#ifdef HAVE_WINTERNL_H
+#include <winternl.h>
+#else
+
+#ifdef HAVE_WINBASE_H
+#include <winbase.h>
+#endif
+#ifdef HAVE_NTDEF_H
+#include <ntdef.h>
+#endif
+#ifdef HAVE_DDK_NTDDK_H
+#include <ddk/ntddk.h>
+#endif
+
+#endif
+HANDLE fh;
+IO_STATUS_BLOCK io;
+NTSTATUS status;
+long buf[66];
+
+NTSTATUS (NTAPI *pNtQueryInformationFile) (HANDLE, PIO_STATUS_BLOCK, PVOID,
+	 				   ULONG, FILE_INFORMATION_CLASS);
+pNtQueryInformationFile = (NTSTATUS (NTAPI *)(HANDLE, PIO_STATUS_BLOCK,
+					      PVOID, ULONG, FILE_INFORMATION_CLASS))
+	GetProcAddress (GetModuleHandle ("ntdll.dll"), "NtQueryInformationFile");
+pNtQueryInformationFile (fh, &io, pfni, sizeof buf, FileNameInformation);
+
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+
+else
+
+$as_echo "#define HAVE_NTQUERYINFORMATIONFILE 1" >>confdefs.h
+
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+
+
 if test ${build} = ${host} -a ${host} = ${target} ; then
    case ${host_os} in
    solaris*)
diff --git a/gdb/configure.ac b/gdb/configure.ac
index 48f37c8..63284f3 100644
--- a/gdb/configure.ac
+++ b/gdb/configure.ac
@@ -1121,6 +1121,13 @@ AC_CHECK_HEADERS(term.h, [], [],
 #endif
 ])
 
+# Check header winternl.h, if not found, check ntdef.h winbase.h and ddk/ntddk.h.
+case "${host}" in
+  *-*-mingw*) AC_CHECK_HEADERS(winternl.h, [],
+  	      	[AC_CHECK_HEADERS(ntdef.h winbase.h ddk/ntddk.h, [], [],[])], [])
+		;;
+esac
+
 # ------------------------- #
 # Checks for declarations.  #
 # ------------------------- #
@@ -1715,6 +1722,39 @@ fi
 AC_SUBST(RDYNAMIC)
 AC_MSG_RESULT($found)
 
+
+
+AC_TRY_COMPILE([#include <windows.h>], [
+#ifdef HAVE_WINTERNL_H
+#include <winternl.h>
+#else
+
+#ifdef HAVE_WINBASE_H
+#include <winbase.h>
+#endif
+#ifdef HAVE_NTDEF_H
+#include <ntdef.h>
+#endif
+#ifdef HAVE_DDK_NTDDK_H
+#include <ddk/ntddk.h>
+#endif
+
+#endif
+HANDLE fh;
+IO_STATUS_BLOCK io;
+NTSTATUS status;
+long buf[66];
+
+NTSTATUS (NTAPI *pNtQueryInformationFile) (HANDLE, PIO_STATUS_BLOCK, PVOID,
+	 				   ULONG, FILE_INFORMATION_CLASS);
+pNtQueryInformationFile = (NTSTATUS (NTAPI *)(HANDLE, PIO_STATUS_BLOCK,
+					      PVOID, ULONG, FILE_INFORMATION_CLASS))
+	GetProcAddress (GetModuleHandle ("ntdll.dll"), "NtQueryInformationFile");
+pNtQueryInformationFile (fh, &io, pfni, sizeof buf, FileNameInformation);
+
+], [],
+[AC_DEFINE(HAVE_NTQUERYINFORMATIONFILE, 1, [Define if function NtQueryInformationFile can be used.])])
+
 dnl For certain native configurations, we need to check whether thread
 dnl support can be built in or not.
 dnl
diff --git a/gdb/defs.h b/gdb/defs.h
index 014d7d4..b226157 100644
--- a/gdb/defs.h
+++ b/gdb/defs.h
@@ -790,6 +790,10 @@ enum block_enum
   FIRST_LOCAL_BLOCK = 2
 };
 
+#ifdef __MINGW32__
+int is_in_cygwin_p (void);
+#endif
+
 #include "utils.h"
 
 #endif /* #ifndef DEFS_H */
diff --git a/gdb/mingw-hdep.c b/gdb/mingw-hdep.c
index efc9848..2ba8017 100644
--- a/gdb/mingw-hdep.c
+++ b/gdb/mingw-hdep.c
@@ -28,6 +28,22 @@
 #include "readline/readline.h"
 
 #include <windows.h>
+#include <wchar.h>
+
+#ifdef HAVE_WINTERNL_H
+#include <winternl.h>
+#else
+#ifdef HAVE_WINBASE_H
+#include <winbase.h>
+#endif
+#ifdef HAVE_NTDEF_H
+#include <ntdef.h>
+#endif
+#ifdef HAVE_DDK_NTDDK_H
+#include <ddk/ntddk.h>
+#endif
+
+#endif /* HAVE_WINTERNL_H */
 
 /* This event is signalled whenever an asynchronous SIGINT handler
    needs to perform an action in the main thread.  */
@@ -265,6 +281,80 @@ gdb_call_async_signal_handler (struct async_signal_handler *handler,
   SetEvent (sigint_event);
 }
 
+#if HAVE_NTQUERYINFORMATIONFILE
+
+/* Return the file name of handle FH.  */
+
+static PWCHAR
+get_filename_from_handle (HANDLE fh)
+{
+  IO_STATUS_BLOCK io;
+  NTSTATUS status;
+  long buf[66];	/* NAME_MAX + 1 + sizeof ULONG */
+  PFILE_NAME_INFORMATION pfni = (PFILE_NAME_INFORMATION) buf;
+  static NTSTATUS (NTAPI *pNtQueryInformationFile) (HANDLE,
+						    PIO_STATUS_BLOCK,
+						    PVOID, ULONG,
+						    FILE_INFORMATION_CLASS);
+
+  /* Calling the native NT function NtQueryInformationFile is required to
+     support pre-Vista systems.  If that's of no concern, Vista introduced
+     the GetFileInformationByHandleEx call with the FileNameInfo info class,
+     which can be used instead. */
+  if (!pNtQueryInformationFile)
+    {
+      pNtQueryInformationFile = (NTSTATUS (NTAPI *)(HANDLE, PIO_STATUS_BLOCK,
+						    PVOID, ULONG, FILE_INFORMATION_CLASS))
+	GetProcAddress (GetModuleHandle ("ntdll.dll"),
+			"NtQueryInformationFile");
+      if (pNtQueryInformationFile == NULL)
+	return NULL;
+    }
+  if (!NT_SUCCESS (pNtQueryInformationFile (fh, &io, pfni, sizeof buf,
+					   FileNameInformation)))
+    return NULL;
+
+  /* The filename is not guaranteed to be NUL-terminated. */
+  pfni->FileName[pfni->FileNameLength / sizeof (WCHAR)] = L'\0';
+
+  return pfni->FileName;
+}
+
+#endif /* HAVE_NTQUERYINFORMATIONFILE */
+
+/* Return true if GDB is running in Cygwin with or without pseudo-tty.  */
+
+int
+is_in_cygwin_p (void)
+{
+  PWCHAR cp;
+  /* Now fetch the underlying HANDLE of stdin. */
+  HANDLE fh = (HANDLE) _get_osfhandle (fileno (stdin));
+
+  if (!fh || fh == INVALID_HANDLE_VALUE)
+    return 0;
+
+#ifdef HAVE_NTQUERYINFORMATIONFILE
+  cp = get_filename_from_handle (fh);
+#else
+  cp = NULL;
+#endif
+
+  /* Now check the name pattern.  With pseudo-tty, the filename of
+     handle of stdin looks like this:
+
+       \cygwin-c5e39b7a9d22bafb-{p,t}ty1-from-master
+
+     Without pseudo-tty, the filename of handle of stdin looks like this:
+
+       \cygwin-c5e39b7a9d22bafb-pipe-0x14C8-0x3
+
+     If the file name is prefixed with "\cygwin-", GDB is running in
+     cygwin.  */
+
+  return (cp != NULL && wcsncmp (cp, L"\\cygwin-", 8) == 0);
+}
+
 /* -Wmissing-prototypes */
 extern initialize_file_ftype _initialize_mingw_hdep;
 
-- 
1.7.7.6