memory leak, buffer-size-calculation overflow in error.c

[Paul Eggert <eggert at CS dot UCLA dot EDU>]

While auditing gnulib code for buffer size calculation overflows, I
noticed that misc/error.c had two such problems, along with a memory leak.
Here is a proposed patch.

2003-09-26  Paul Eggert  <eggert@twinsun.com>

	* error.c (SIZE_MAX) [!defined SIZE_MAX]: Define.
	(error_tail): Do not loop, reallocating temporary buffer, since
	the output cannot contain more wide characters than the input
	contains bytes, the size must be big enough already.  This avoids
	one potential size overflow calculation.  Check for size overflow
	when calculating temporary buffer size.  Free temporary buffer
	when done, if it was allocated with malloc; this plugs a memory
	leak.  Remove casts from void * to pointers, that are no longer
	needed now that we're assuming C89 or better.

--- misc/error.c	Wed Sep  3 13:11:51 2003
+++ misc/error.c-fix	Fri Sep 26 00:49:03 2003
@@ -103,6 +103,10 @@ extern void __error_at_line (int status,
 char *strerror_r ();
 # endif
 
+#ifndef SIZE_MAX
+# define SIZE_MAX ((size_t) -1)
+#endif
+
 /* The calling program should define program_name and set it to the
    name of the executing program.  */
 extern char *program_name;
@@ -178,40 +182,27 @@ error_tail (int status, int errnum, cons
     {
 #   define ALLOCA_LIMIT	2000
       size_t len = strlen (message) + 1;
-      wchar_t *wmessage = NULL;
+      const wchar_t *wmessage = L"out of memory";
       mbstate_t st;
       size_t res;
       const char *tmp;
+      wchar_t *wbuf = (len < ALLOCA_LIMIT
+		       ? (void *) alloca (len * sizeof *wbuf)
+		       : len <= SIZE_MAX / sizeof *wbuf
+		       ? malloc (len * sizeof *wbuf)
+		       : NULL);
 
-      do
+      if (wbuf)
 	{
-	  if (len < ALLOCA_LIMIT)
-	    wmessage = (wchar_t *) alloca (len * sizeof (wchar_t));
-	  else
-	    {
-	      if (wmessage != NULL && len / 2 < ALLOCA_LIMIT)
-		wmessage = NULL;
-
-	      wmessage = (wchar_t *) realloc (wmessage,
-					      len * sizeof (wchar_t));
-
-	      if (wmessage == NULL)
-		{
-		  fputws_unlocked (L"out of memory\n", stderr);
-		  return;
-		}
-	    }
-
 	  memset (&st, '\0', sizeof (st));
 	  tmp =message;
+	  res = mbsrtowcs (wbuf, &tmp, len, &st);
+	  wmessage = res == (size_t) -1 ? L"???" : wbuf;
 	}
-      while ((res = mbsrtowcs (wmessage, &tmp, len, &st)) == len);
-
-      if (res == (size_t) -1)
-	/* The string cannot be converted.  */
-	wmessage = (wchar_t *) L"???";
 
       __vfwprintf (stderr, wmessage, args);
+      if (! (len < ALLOCA_LIMIT))
+	free (wbuf);
     }
   else
 #  endif