[PATCH] Fixup convenience variables on endian switch

[Andrew STUBBS <andrew dot stubbs at st dot com>]

Hi,

The attached patch attempts to do the Right Thing for internal variables 
when 'set endian' is used (explicitly or implicitly).

I have not (and cannot) test it with all varieties of host machine and 
type, but I am confident that it works on x86 Linux and sparc Solaris 
for integer (including pointer) and float types.

I have run the regression test suite on i686-pc-linux-gnu native and 
sh-elf cross.

Demonstration:

Before

  (gdb) set $a = 1
  (gdb) set $b = (float)1.1
  (gdb) set $c = (double)1.2
  (gdb) set $d = (long double)1.3
  (gdb) set $e = (long long)10000000
  (gdb) show conv
  $e = 10000000
  $d = 1.3000000000000000444089209850062616
  $c = 1.2
  $b = 1.10000002
  $a = 1
  $trace_frame = -1
  $tpnum = 0
  (gdb) show endian
  The target endianness is set automatically (currently little endian)
  (gdb) set endian big
  The target is assumed to be big endian
  (gdb) show conv
  $e = -9180983664580755456
  $d = 1.3000000000000000444089209850062616
  $c = 1.2
  $b = 1.10000002
  $a = 16777216
  $trace_frame = -1
  $tpnum = 0
  (gdb) set endian little
  The target is assumed to be little endian
  (gdb) show conv
  $e = 10000000
  $d = 1.3000000000000000444089209850062616
  $c = 1.2
  $b = 1.10000002
  $a = 1
  $trace_frame = -1
  $tpnum = 0

After

  (gdb) set $a = 1
  (gdb) set $b = (float)1.1
  (gdb) set $c = (double)1.2
  (gdb) set $d = (long double)1.3
  (gdb) set $e = (long long)10000000
  (gdb) show conv
  $e = 10000000
  $d = 1.3000000000000000444089209850062616
  $c = 1.2
  $b = 1.10000002
  $a = 1
  $trace_frame = -1
  $tpnum = 0
  (gdb) show endian
  The target endianness is set automatically (currently little endian)
  (gdb) set endian big
  The target is assumed to be big endian
  (gdb) show conv
  $e = 10000000
  $d = 1.3000000000000000444089209850062616
  $c = 1.2
  $b = 1.10000002
  $a = 1
  $trace_frame = -1
  $tpnum = 0
  (gdb) set endian little
  The target is assumed to be little endian
  (gdb) show conv
  $e = 10000000
  $d = 1.3000000000000000444089209850062616
  $c = 1.2
  $b = 1.10000002
  $a = 1
  $trace_frame = -1
  $tpnum = 0

Andrew Stubbs
2005-11-16  Andrew Stubbs  <andrew.stubbs@st.com>

	* arch-utils.c: Include value.h.
	(set_endian): Call fixup_all_internalvars_endian().
	* value.h (struct internalvar): Add endian field.
	(fixup_all_internalvars_endian): New prototype.
	* value.c (fixup_internalvar_endian): New function.
	(fixup_all_internalvars_endian): New function.
	(lookup_internalvar): Initialize endian field.
	(set_internalvar): Reset endian field.
	* Makefile.in (arch-utils.o): Add value.h to dependencies.


Index: src/gdb/arch-utils.c
===================================================================
--- src.orig/gdb/arch-utils.c	2005-11-07 11:50:22.000000000 +0000
+++ src/gdb/arch-utils.c	2005-11-07 16:40:56.000000000 +0000
@@ -32,6 +32,7 @@
 #include "sim-regno.h"
 #include "gdbcore.h"
 #include "osabi.h"
+#include "value.h"
 
 #include "version.h"
 
@@ -417,6 +418,8 @@
     internal_error (__FILE__, __LINE__,
 		    _("set_endian: bad value"));
   show_endian (gdb_stdout, from_tty, NULL, NULL);
+
+  fixup_all_internalvars_endian ();
 }
 
 /* Functions to manipulate the architecture of the target */
Index: src/gdb/value.h
===================================================================
--- src.orig/gdb/value.h	2005-11-07 11:50:22.000000000 +0000
+++ src/gdb/value.h	2005-11-07 16:40:56.000000000 +0000
@@ -245,6 +245,7 @@
   struct internalvar *next;
   char *name;
   struct value *value;
+  int endian;
 };
 
 
@@ -416,6 +417,8 @@
 
 extern struct internalvar *lookup_internalvar (char *name);
 
+extern void fixup_all_internalvars_endian (void);
+
 extern int value_equal (struct value *arg1, struct value *arg2);
 
 extern int value_less (struct value *arg1, struct value *arg2);
Index: src/gdb/value.c
===================================================================
--- src.orig/gdb/value.c	2005-11-07 16:40:54.000000000 +0000
+++ src/gdb/value.c	2005-11-07 16:40:56.000000000 +0000
@@ -768,6 +768,49 @@
 }
 
 
+/* Reverse the endianess of the variable.
+   This is used when the the endianness is changed with 'set endian'
+   or when a new file of opposite endianess is loaded.  */
+
+static void
+fixup_internalvar_endian (struct internalvar *var)
+{
+  int i, j;
+  gdb_byte temp;
+  gdb_byte *array = value_contents_raw (var->value);
+
+  /* Don't do anything if the endian has not changed.
+     Also disregard FP types because they don't seem to vary with
+     endian - at least, not on i686 Linux or sparc Solaris.  */
+  if (var->endian != TARGET_BYTE_ORDER
+      && TYPE_CODE (value_type (var->value)) != TYPE_CODE_FLT)
+    {
+      /* Reverse the bytes.
+	 This may not be the right thing for some types
+	 so don't be afraid of messing with it if you encounter problems. */
+      for (i=0,j=TYPE_LENGTH (var->value->enclosing_type)-1; i<j; i++,j--)
+	{
+	  temp = array[j];
+	  array[j] = array[i];
+	  array[i] = temp;
+	}
+      var->endian = TARGET_BYTE_ORDER;
+    }
+}
+
+
+/* Swap the endianness of all the internal variables. */
+
+void
+fixup_all_internalvars_endian (void)
+{
+  struct internalvar *iv;
+
+  for (iv=internalvars; iv!=NULL; iv=iv->next)
+    fixup_internalvar_endian (iv);
+}
+
+
 /* Look up an internal variable with name NAME.  NAME should not
    normally include a dollar sign.
 
@@ -786,6 +829,7 @@
   var = (struct internalvar *) xmalloc (sizeof (struct internalvar));
   var->name = concat (name, (char *)NULL);
   var->value = allocate_value (builtin_type_void);
+  var->endian = TARGET_BYTE_ORDER;
   release_value (var->value);
   var->next = internalvars;
   internalvars = var;
@@ -840,6 +884,7 @@
      long.  */
   xfree (var->value);
   var->value = newval;
+  var->endian = TARGET_BYTE_ORDER;
   release_value (newval);
   /* End code which must not call error().  */
 }
Index: src/gdb/Makefile.in
===================================================================
--- src.orig/gdb/Makefile.in	2005-11-07 16:40:46.000000000 +0000
+++ src/gdb/Makefile.in	2005-11-07 16:45:08.000000000 +0000
@@ -1732,7 +1732,7 @@
 arch-utils.o: arch-utils.c $(defs_h) $(arch_utils_h) $(buildsym_h) \
 	$(gdbcmd_h) $(inferior_h) $(gdb_string_h) $(regcache_h) \
 	$(gdb_assert_h) $(sim_regno_h) $(gdbcore_h) $(osabi_h) $(version_h) \
-	$(floatformat_h)
+	$(floatformat_h) $(value_h)
 arm-linux-nat.o: arm-linux-nat.c $(defs_h) $(inferior_h) $(gdbcore_h) \
 	$(gdb_string_h) $(regcache_h) $(arm_tdep_h) $(gregset_h) \
 	$(target_h) $(linux_nat_h)