[PATCH 25/25] Remove macros extend_alloca, extend_alloca_account [BZ #18023]

[Florian Weimer <fweimer at redhat dot com>]

And also the helper macro stackinfo_alloca_round.

extend_alloca simply does not work on x86_64 and current i386 because
its peculiar stack alignment rules.

Here's an analysis of the _dl_fini situation (before the removal of
extend_alloca).

Dump of assembler code for function _dl_fini:
<+0>:	push   %rbp
<+1>:	mov    %rsp,%rbp
<+4>:	push   %r15
<+6>:	push   %r14
<+8>:	push   %r13
<+10>:	push   %r12
<+12>:	push   %rbx
<+13>:	sub    $0x38,%rsp

The function pushes 6 registers on the stack and allocates 0x38 bytes,
which means that %rsp is a multiple of 16 after function prologue.

The initial alloca allocation does not change %rsp alignment:

<+210>:	shr    $0x4,%rcx
<+214>:	shl    $0x4,%rcx
<+218>:	sub    %rcx,%rsp

%r15 is the address of the previous stack allocation, it is used below.

This is the extend_alloca reallocation branch:

<+734>:	add    $0xf,%rdx
<+738>:	and    $0xfffffffffffffff0,%rdx
<+742>:	lea    0x1e(%rdx),%rcx
<+746>:	shr    $0x4,%rcx
<+750>:	shl    $0x4,%rcx
<+754>:	sub    %rcx,%rsp
<+757>:	lea    0xf(%rsp),%rcx
<+762>:	and    $0xfffffffffffffff0,%rcx
<+766>:	lea    (%rcx,%rdx,1),%rsi
<+770>:	cmp    %rsi,%r15
<+773>:	je     0x7f963940b673 <_dl_fini+787>
<+775>:	mov    %rdx,-0x58(%rbp)
<+787>:	add    %rdx,-0x58(%rbp)

(a) %rdx, the new requested size, is rounded up to a multiple of 16
(+734, %+738), and the result is stored in %rdx@738.

(b) %rdx@738 + 31 is rounded down to a multiple of 16, the result is
stored in rcx@750 (+742, +746, +750).  So %rcx@750 == %rdx@738 + 16.

(c) %rcx@750 bytes are allocated on the stack (+754).  %rsp is rounded
upwards to a multiple of 16, result is stored in %rcx@762 (+757, +762).
This does not change the value of %rsp because it already was a multiple
of 16.

(d) %rsi@766 == %rcx@762 + %rdx@738 is compared against %r15.  But this
comparison is always false because we allocated 16 extra bytes on the
stack in (b), which were reserved for the alignment in (c), but in fact
unused.  We are left with a gap in stack usage, and the comparison is
always false.

(@XXX refers to register values after executing the instruction at
offset +XXX.)

If the alignment gap was actually used because of different alignment
for %rsp, then the comparison failure would still occur because the gap
would not have been added after this reallocation, but before the
previous allocation.

As a result, extend_alloca is never able to merge allocations.  It also
turns out that the interface is difficult to use, especially in
cojunction with alloca account (which is rarely optional).
---
 include/alloca.h | 40 ----------------------------------------
 1 file changed, 40 deletions(-)

diff --git a/include/alloca.h b/include/alloca.h
index 0150025..f190b87 100644
--- a/include/alloca.h
+++ b/include/alloca.h
@@ -20,57 +20,17 @@ libc_hidden_proto (__libc_alloca_cutoff)
 
 #include <allocalim.h>
 
-#ifndef stackinfo_alloca_round
-# define stackinfo_alloca_round(l) (((l) + 15) & -16)
-#endif
-
-#if _STACK_GROWS_DOWN
-# define extend_alloca(buf, len, newlen) \
-  (__typeof (buf)) ({ size_t __newlen = stackinfo_alloca_round (newlen);      \
-		      char *__newbuf = __alloca (__newlen);		      \
-		      if (__newbuf + __newlen == (char *) (buf))	      \
-			len += __newlen;				      \
-		      else						      \
-			len = __newlen;					      \
-		      __newbuf; })
-#elif _STACK_GROWS_UP
-# define extend_alloca(buf, len, newlen) \
-  (__typeof (buf)) ({ size_t __newlen = stackinfo_alloca_round (newlen);      \
-		      char *__newbuf = __alloca (__newlen);		      \
-		      char *__buf = (char *) (buf);			      \
-		      if (__buf + len == __newbuf)			      \
-			{						      \
-			  len += __newlen;				      \
-			  __newbuf = __buf;				      \
-			}						      \
-		      else						      \
-			len = __newlen;					      \
-		      __newbuf; })
-#else
-# define extend_alloca(buf, len, newlen) \
-  __alloca (((len) = (newlen)))
-#endif
-
 #if defined stackinfo_get_sp && defined stackinfo_sub_sp
 # define alloca_account(size, avar) \
   ({ void *old__ = stackinfo_get_sp ();					      \
      void *m__ = __alloca (size);					      \
      avar += stackinfo_sub_sp (old__);					      \
      m__; })
-# define extend_alloca_account(buf, len, newlen, avar) \
-  ({ void *old__ = stackinfo_get_sp ();					      \
-     void *m__ = extend_alloca (buf, len, newlen);			      \
-     avar += stackinfo_sub_sp (old__);					      \
-     m__; })
 #else
 # define alloca_account(size, avar) \
   ({ size_t s__ = (size);						      \
      avar += s__;							      \
      __alloca (s__); })
-# define extend_alloca_account(buf, len, newlen, avar) \
-  ({ size_t s__ = (newlen);						      \
-     avar += s__;							      \
-     extend_alloca (buf, len, s__); })
 #endif
 
 #endif
-- 
2.1.0