[RFC] Gdb does not honor -tty option with builtin simulators (includes possible patch)

[Fred Fish <fnf at ninemoons dot com>]

For programs run inside gdb's internal simulator, gdb does not honor
the -tty command line option (or the gdb "tty" command) to send the
target's stdout and stderr streams to a specified tty.

One place this shows up as a problem is when running gdb from eclipse,
where eclipse starts gdb with a specific -tty option in order to
capture the target stdout/stderr streams and display the output in the
eclipse console window.

Here is a typescript that demonstrates the problem, using arm-elf-gdb
and attempting to send the target stdout/stderr to another xterm window.
Instead, the output shows up in gdb's stderr (stdout?) stream.

Also included below is a somewhat hackish patch that does fix the
problem, though may not be the optimal long term solution.  Comments
on how to fix this properly would be appreciated as this is a part of
gdb I'm not intimately familiar with.

This is in the gdb bugs database as sim/1538

-Fred

============================= typescript ============================

Script started on Fri 30 Jan 2004 10:22:19 AM MST
$ cat hello.c
#include <stdio.h>
main ()
{
  fprintf (stdout, "This message to stdout on fileno %d\n", fileno(stdout));
  fflush (stdout);
  fprintf (stderr, "This message to stderr on fileno %d\n", fileno(stderr));
  fflush(stdout);
}
$ ./gdb -nw -nx -tty=/dev/pts/2 hello
GNU gdb 2004-01-30-cvs
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB.  Type "show warranty" for details.
This GDB was configured as "--host=i686-pc-linux-gnu --target=arm-elf"...
(gdb) tar sim
Connected to the simulator.
(gdb) load
Loading section .init, size 0x1c vma 0x8000
Loading section .text, size 0x8e54 vma 0x801c
Loading section .fini, size 0x18 vma 0x10e70
Loading section .rodata, size 0x358 vma 0x10e88
Loading section .data, size 0x8bc vma 0x112e0
Loading section .eh_frame, size 0x4 vma 0x11b9c
Loading section .ctors, size 0x8 vma 0x11ba0
Loading section .dtors, size 0x8 vma 0x11ba8
Loading section .jcr, size 0x4 vma 0x11bb0
Start address 0x811c
Transfer rate: 316832 bits in <1 sec.
(gdb) run
Starting program: /links1/build/sourceware/gdb/T-arm-elf/gdb/hello 
This message to stdout on fileno 1
This message to stderr on fileno 2

Program exited normally.
(gdb) quit
$ exit
exit

Script done on Fri 30 Jan 2004 10:23:10 AM MST


===========================  Hackish patch ==============================

(part of this patch was already submitted to the gdb-patches list)

        2004-01-29  Fred Fish  <fnf@redhat.com>

	* main.c (gdb_stdtarg): Move definition to group with other
	gdb_stdtarg definitions.
	* remote-sim.c (gdb_os_write_stderr): Write output to
	gdb_stdtargerr stream instead of gdb_stdtarg stream.
	(gdb_os_flush_stderr): Flush gdb_stdtargerr steam instead of
	gdb_stderr stream.

	* infcmd.c (fcntl.h): Include for tty_command use.
	(tty_command): Initialize gdb_stdtarg and gdb_stdtargerr streams
	as set by "-tty" command line option or "tty" command.

Index: infcmd.c
===================================================================
RCS file: /cvs/src/src/gdb/infcmd.c,v
retrieving revision 1.104
diff -c -p -r1.104 infcmd.c
*** infcmd.c	26 Jan 2004 20:52:10 -0000	1.104
--- infcmd.c	30 Jan 2004 17:19:33 -0000
***************
*** 45,50 ****
--- 45,51 ----
  #include "block.h"
  #include <ctype.h>
  #include "gdb_assert.h"
+ #include <fcntl.h>	/* Part of gdb_stdtarg/gdb_stdtargerr hack */
  
  /* Functions exported for general use, in inferior.h: */
  
*************** tty_command (char *file, int from_tty)
*** 377,382 ****
--- 378,404 ----
      error_no_arg ("terminal name for running target process");
  
    inferior_io_terminal = savestring (file, strlen (file));
+ 
+   {
+     /* Hack to make sure target stdio and stderr streams go where
+        directed by the "-tty" command line option or the "tty"
+        command. */
+     int tty;
+     FILE *fp;
+ #ifdef USE_O_NOCTTY
+     tty = open (inferior_io_terminal, O_RDWR | O_NOCTTY);
+ #else
+     tty = open (inferior_io_terminal, O_RDWR);
+ #endif
+     if (tty == -1)
+       {
+ 	print_sys_errmsg (inferior_io_terminal, errno);
+ 	return;
+       }
+     fp = fdopen (tty, "r+");
+     gdb_stdtarg = stdio_fileopen (fp);
+     gdb_stdtargerr = stdio_fileopen (fp);
+   }
  }
  
  static void
Index: main.c
===================================================================
RCS file: /cvs/src/src/gdb/main.c,v
retrieving revision 1.37
diff -c -p -r1.37 main.c
*** main.c	19 Jan 2004 19:56:01 -0000	1.37
--- main.c	30 Jan 2004 17:19:35 -0000
*************** char *gdb_sysroot = 0;
*** 72,81 ****
  struct ui_file *gdb_stdout;
  struct ui_file *gdb_stderr;
  struct ui_file *gdb_stdlog;
- struct ui_file *gdb_stdtarg;
  struct ui_file *gdb_stdin;
  /* target IO streams */
  struct ui_file *gdb_stdtargin;
  struct ui_file *gdb_stdtargerr;
  
  /* Whether to enable writing into executable and core files */
--- 72,81 ----
  struct ui_file *gdb_stdout;
  struct ui_file *gdb_stderr;
  struct ui_file *gdb_stdlog;
  struct ui_file *gdb_stdin;
  /* target IO streams */
  struct ui_file *gdb_stdtargin;
+ struct ui_file *gdb_stdtarg;
  struct ui_file *gdb_stdtargerr;
  
  /* Whether to enable writing into executable and core files */
Index: remote-sim.c
===================================================================
RCS file: /cvs/src/src/gdb/remote-sim.c,v
retrieving revision 1.33
diff -c -p -r1.33 remote-sim.c
*** remote-sim.c	13 Nov 2003 19:06:26 -0000	1.33
--- remote-sim.c	30 Jan 2004 17:19:36 -0000
*************** gdb_os_write_stderr (host_callback *p, c
*** 221,227 ****
      {
        b[0] = buf[i];
        b[1] = 0;
!       fputs_unfiltered (b, gdb_stdtarg);
      }
    return len;
  }
--- 221,227 ----
      {
        b[0] = buf[i];
        b[1] = 0;
!       fputs_unfiltered (b, gdb_stdtargerr);
      }
    return len;
  }
*************** gdb_os_write_stderr (host_callback *p, c
*** 231,237 ****
  static void
  gdb_os_flush_stderr (host_callback *p)
  {
!   gdb_flush (gdb_stderr);
  }
  
  /* GDB version of printf_filtered callback.  */
--- 231,237 ----
  static void
  gdb_os_flush_stderr (host_callback *p)
  {
!   gdb_flush (gdb_stdtargerr);
  }
  
  /* GDB version of printf_filtered callback.  */