This is the mail archive of the
libc-alpha@sourceware.org
mailing list for the glibc project.
[PATCH] PPC32 performance improvements for sqrt
- From: Steven Munroe <munroesj at us dot ibm dot com>
- To: libc-alpha at sources dot redhat dot com, Ryan Arnold <rsa at us dot ibm dot com>
- Date: Thu, 01 Nov 2007 17:38:44 -0500
- Subject: [PATCH] PPC32 performance improvements for sqrt
For POWER4 and later, the performance of sqrt()/sqrtf() is limited by the overhead of stacking a frame, addressing the GOT, testing for NaN etc, which is only need in case we need to report EDOM via __kernel_standard(). For PowerPC the fsqrt[s] instruction does all the work except for setting errno to EDOM for non-zero negative values.
So the attached patch simulates prologue shrink-wrapping such that the fsqrt calculation and basic error checking occurs before the prologure. If there are no errors the result is returned directly. Otherwise stack stack a frame and reports errors if needed via __kernel_standard(). The 32-bit version is written in asm because of bugs in gcc that insistent stacking frames even for leaf routines.
This patch only impacts builds using:
--with-cpu=[power4,970,power5,power5+,power6,power6x]
the code for power4/970 is slightly different from that for power5 and later due to increased latency accessing the FPSCR. For micro benchmarks I see the following improvement:
1.53 X on ppc970 2.0GHz
1.73 X on power4 1.0GHz
1.94 X on power5 1.9GHz
1.66 X on power6 4.2GHz
This patch will also be release with powerpc-cpu-V0.07.
2007-10-26 Steven Munroe <sjmunroe@us.ibm.com>
* sysdeps/powerpc/powerpc32/power4/fpu/w_sqrt.S: New File.
* sysdeps/powerpc/powerpc32/power4/fpu/w_sqrt.c: Removed.
* sysdeps/powerpc/powerpc32/power4/fpu/w_sqrtf.S: New File.
* sysdeps/powerpc/powerpc32/power4/fpu/w_sqrtf.c: Removed.
* sysdeps/powerpc/powerpc32/power5/fpu/w_sqrt.S: New File.
* sysdeps/powerpc/powerpc32/power5/fpu/w_sqrtf.S: New File.
diff -urN libc25-cvstip-20070919/sysdeps/powerpc/powerpc32/power4/fpu/w_sqrt.S libc25/sysdeps/powerpc/powerpc32/power4/fpu/w_sqrt.S
--- libc25-cvstip-20070919/sysdeps/powerpc/powerpc32/power4/fpu/w_sqrt.S Wed Dec 31 18:00:00 1969
+++ libc25/sysdeps/powerpc/powerpc32/power4/fpu/w_sqrt.S Wed Oct 31 16:22:57 2007
@@ -0,0 +1,119 @@
+/* sqrt function. PowerPC32 version.
+ Copyright (C) 2007 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, write to the Free
+ Software Foundation, Inc., 1 Franklin Street, Fifth Floor, Boston MA
+ 02110-1301 USA. */
+
+#include <sysdep.h>
+#include <math_ldbl_opt.h>
+
+/* double [fp1] sqrt (double x [fp1])
+ Power4 (ISA V2.0) and above implement sqrt in hardware (not optional).
+ The fsqrt instruction generates the correct value for all inputs and
+ sets the appropriate floating point exceptions. Extented checking is
+ only needed to set errno (via __kernel_standard) if the input value
+ is negative.
+
+ The fsqrt will set FPCC and FU (Floating Point Unordered or NaN
+ to indicated that the input value was negative or NaN. Use Move to
+ Condition Register from FPSCR to copy the FPCC field to cr1. The
+ branch on summary overflow transfers control to w_sqrt to process
+ any error conditions. Otherwise we can return the result directly.
+
+ This part of the function is a leaf routine, so no need to stack a
+ frame or execute prologue/epilogue code. This means it is safe to
+ transfer directly to w_sqrt as long as the input value (f1) is
+ preserved. Putting the the sqrt result into f2 (double parameter 2)
+ allows passing both the input value and sqrt result into the extended
+ wrapper so there is no need to recompute.
+
+ This tactic avoids the overhead of stacking a frame for the normal
+ (non-error) case. Until gcc supports prologue shrink-wrapping
+ this is the best we can do. */
+
+ .section ".text"
+ .machine power4
+EALIGN (__sqrt, 5, 0)
+ fsqrt fp2,fp1
+ mcrfs cr1,4
+ bso- cr1,.Lw_sqrt
+ fmr fp1,fp2
+ blr
+ .align 4
+.Lw_sqrt:
+ mflr r0
+ stwu r1,-16(r1)
+ cfi_adjust_cfa_offset(16)
+ fmr fp12,fp2
+ stw r0,20(r1)
+ stw r30,8(r1)
+ cfi_offset(lr,20)
+ cfi_offset(r30,8)
+#ifdef SHARED
+# ifdef HAVE_ASM_PPC_REL16
+ bcl 20,31,.LCF1
+.LCF1:
+ mflr r30
+ addis r30,r30,_GLOBAL_OFFSET_TABLE_-.LCF1@ha
+ addi r30,r30,_GLOBAL_OFFSET_TABLE_-.LCF1@l
+ lwz r9,_LIB_VERSION@got(30)
+ lwz r0,0(r9)
+# else
+ bl _GLOBAL_OFFSET_TABLE_@local-4
+ mflr r30
+ lwz r9,_LIB_VERSION@got(30)
+ lwz r0,0(r9)
+# endif
+#else
+ lis r9,_LIB_VERSION@ha
+ lwz r0,_LIB_VERSION@l(r9)
+#endif
+/* if (_LIB_VERSION == _IEEE_) return z; */
+ cmpwi cr7,r0,-1
+ beq- cr7,.L4
+/* if (x != x) return z; !isnan*/
+ fcmpu cr7,fp1,fp1
+ bne- cr7,.L4
+/* if (x < 0.0)
+ return __kernel_standard (x, x, 26) */
+ fmr fp2,fp1
+ fabs fp0,fp1
+ li r3,26
+ fcmpu cr7,fp1,fp0
+ bne- cr7,.L11
+.L4:
+ lwz r0,20(r1)
+ fmr fp1,fp12
+ lwz r30,8(r1)
+ addi r1,r1,16
+ mtlr r0
+ blr
+.L11:
+ bl __kernel_standard@plt
+ fmr fp12,fp1
+ b .L4
+ END (__sqrt)
+
+weak_alias (__sqrt, sqrt)
+
+#ifdef NO_LONG_DOUBLE
+weak_alias (__sqrt, sqrtl)
+strong_alias (__sqrt, __sqrtl)
+#endif
+#if LONG_DOUBLE_COMPAT(libm, GLIBC_2_0)
+compat_symbol (libm, __sqrt, sqrtl, GLIBC_2_0)
+#endif
+
diff -urN libc25-cvstip-20070919/sysdeps/powerpc/powerpc32/power4/fpu/w_sqrt.c libc25/sysdeps/powerpc/powerpc32/power4/fpu/w_sqrt.c
--- libc25-cvstip-20070919/sysdeps/powerpc/powerpc32/power4/fpu/w_sqrt.c Thu Oct 18 17:21:37 2007
+++ libc25/sysdeps/powerpc/powerpc32/power4/fpu/w_sqrt.c Wed Dec 31 18:00:00 1969
@@ -1,62 +0,0 @@
-/* Double-precision floating point square root wrapper.
- Copyright (C) 2004, 2007 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, write to the Free
- Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
- 02111-1307 USA. */
-
-#include <math_ldbl_opt.h>
-#include "math.h"
-#include "math_private.h"
-#include <fenv_libc.h>
-
-#ifdef __STDC__
-double
-__sqrt (double x) /* wrapper sqrt */
-#else
-double
-__sqrt (x) /* wrapper sqrt */
- double x;
-#endif
-{
- double z;
-/* Power4 (ISA V2.0) and above implement sqrt in hardware. */
- __asm __volatile (
- " fsqrt %0,%1\n"
- : "=f" (z)
- : "f" (x));
-#ifdef _IEEE_LIBM
- return z;
-#else
- if (__builtin_expect (_LIB_VERSION == _IEEE_, 0))
- return z;
-
- if (__builtin_expect (x != x, 0))
- return z;
-
- if (__builtin_expect (x < 0.0, 0))
- return __kernel_standard (x, x, 26); /* sqrt(negative) */
- else
- return z;
-#endif
-}
-
-weak_alias (__sqrt, sqrt)
-#ifdef NO_LONG_DOUBLE
- strong_alias (__sqrt, __sqrtl) weak_alias (__sqrt, sqrtl)
-#endif
-#if LONG_DOUBLE_COMPAT(libm, GLIBC_2_0)
-compat_symbol (libm, __sqrt, sqrtl, GLIBC_2_0);
-#endif
diff -urN libc25-cvstip-20070919/sysdeps/powerpc/powerpc32/power4/fpu/w_sqrtf.S libc25/sysdeps/powerpc/powerpc32/power4/fpu/w_sqrtf.S
--- libc25-cvstip-20070919/sysdeps/powerpc/powerpc32/power4/fpu/w_sqrtf.S Wed Dec 31 18:00:00 1969
+++ libc25/sysdeps/powerpc/powerpc32/power4/fpu/w_sqrtf.S Wed Oct 31 16:24:04 2007
@@ -0,0 +1,111 @@
+/* sqrtf function. PowerPC32 version.
+ Copyright (C) 2007 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, write to the Free
+ Software Foundation, Inc., 1 Franklin Street, Fifth Floor, Boston MA
+ 02110-1301 USA. */
+
+#include <sysdep.h>
+#include <math_ldbl_opt.h>
+
+/* float [fp1] sqrts (float x [fp1])
+ Power4 (ISA V2.0) and above implement sqrt in hardware (not optional).
+ The fsqrts instruction generates the correct value for all inputs and
+ sets the appropriate floating point exceptions. Extented checking is
+ only needed to set errno (via __kernel_standard) if the input value
+ is negative.
+
+ The fsqrts will set FPCC and FU (Floating Point Unordered or NaN
+ to indicated that the input value was negative or NaN. Use Move to
+ Condition Register from FPSCR to copy the FPCC field to cr1. The
+ branch on summary overflow transfers control to w_sqrt to process
+ any error conditions. Otherwise we can return the result directly.
+
+ This part of the function is a leaf routine, so no need to stack a
+ frame or execute prologue/epilogue code. This means it is safe to
+ transfer directly to w_sqrt as long as the input value (f1) is
+ preserved. Putting the the sqrt result into f2 (float parameter 2)
+ allows passing both the input value and sqrt result into the extended
+ wrapper so there is no need to recompute.
+
+ This tactic avoids the overhead of stacking a frame for the normal
+ (non-error) case. Until gcc supports prologue shrink-wrapping
+ this is the best we can do. */
+
+ .section ".text"
+ .machine power4
+EALIGN (__sqrtf, 5, 0)
+ fsqrts fp2,fp1
+ mcrfs cr1,4
+ bso- cr1,.Lw_sqrtf
+ fmr fp1,fp2
+ blr
+ .align 4
+.Lw_sqrtf:
+ mflr r0
+ stwu r1,-16(r1)
+ cfi_adjust_cfa_offset(16)
+ fmr fp12,fp2
+ stw r0,20(r1)
+ stw r30,8(r1)
+ cfi_offset(lr,20)
+ cfi_offset(r30,8)
+#ifdef SHARED
+# ifdef HAVE_ASM_PPC_REL16
+ bcl 20,31,.LCF1
+.LCF1:
+ mflr r30
+ addis r30,r30,_GLOBAL_OFFSET_TABLE_-.LCF1@ha
+ addi r30,r30,_GLOBAL_OFFSET_TABLE_-.LCF1@l
+ lwz r9,_LIB_VERSION@got(30)
+ lwz r0,0(r9)
+# else
+ bl _GLOBAL_OFFSET_TABLE_@local-4
+ mflr r30
+ lwz r9,_LIB_VERSION@got(30)
+ lwz r0,0(r9)
+# endif
+#else
+ lis r9,_LIB_VERSION@ha
+ lwz r0,_LIB_VERSION@l(r9)
+#endif
+/* if (_LIB_VERSION == _IEEE_) return z; */
+ cmpwi cr7,r0,-1
+ beq- cr7,.L4
+/* if (x != x, 0) return z; !isnan */
+ fcmpu cr7,fp1,fp1
+ bne- cr7,.L4
+/* if (x < 0.0)
+ return __kernel_standard (x, x, 126) */
+ fmr fp2,fp1
+ fabs fp0,fp1
+ li r3,126
+ fcmpu cr7,1,0
+ bne- cr7,.L11
+.L4:
+ lwz r0,20(r1)
+ fmr fp1,fp12
+ lwz r30,8(r1)
+ addi r1,r1,16
+ mtlr r0
+ blr
+.L11:
+ bl __kernel_standard@plt
+ fmr fp12,fp1
+ b .L4
+ END (__sqrtf)
+
+weak_alias (__sqrtf, sqrtf)
+
diff -urN libc25-cvstip-20070919/sysdeps/powerpc/powerpc32/power4/fpu/w_sqrtf.c libc25/sysdeps/powerpc/powerpc32/power4/fpu/w_sqrtf.c
--- libc25-cvstip-20070919/sysdeps/powerpc/powerpc32/power4/fpu/w_sqrtf.c Thu Oct 18 17:21:37 2007
+++ libc25/sysdeps/powerpc/powerpc32/power4/fpu/w_sqrtf.c Wed Dec 31 18:00:00 1969
@@ -1,60 +0,0 @@
-/* Single-precision floating point square root wrapper.
- Copyright (C) 2004, 2007 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, write to the Free
- Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
- 02111-1307 USA. */
-
-#include "math.h"
-#include "math_private.h"
-#include <fenv_libc.h>
-
-#include <sysdep.h>
-#include <ldsodefs.h>
-
-#ifdef __STDC__
-float
-__sqrtf (float x) /* wrapper sqrtf */
-#else
-float
-__sqrtf (x) /* wrapper sqrtf */
- float x;
-#endif
-{
-#ifdef _IEEE_LIBM
- return __ieee754_sqrtf (x);
-#else
- float z;
-/* Power4 (ISA V2.0) and above implement sqrtf in hardware. */
- __asm __volatile (
- " fsqrts %0,%1\n"
- : "=f" (z)
- : "f" (x));
-
- if (__builtin_expect (_LIB_VERSION == _IEEE_, 0))
- return z;
-
- if (__builtin_expect (x != x, 0))
- return z;
-
- if (__builtin_expect (x < 0.0, 0))
- /* sqrtf(negative) */
- return (float) __kernel_standard ((double) x, (double) x, 126);
- else
- return z;
-#endif
-}
-
-weak_alias (__sqrtf, sqrtf)
diff -urN libc25-cvstip-20070919/sysdeps/powerpc/powerpc32/power5/fpu/w_sqrt.S libc25/sysdeps/powerpc/powerpc32/power5/fpu/w_sqrt.S
--- libc25-cvstip-20070919/sysdeps/powerpc/powerpc32/power5/fpu/w_sqrt.S Wed Dec 31 18:00:00 1969
+++ libc25/sysdeps/powerpc/powerpc32/power5/fpu/w_sqrt.S Wed Oct 31 12:16:52 2007
@@ -0,0 +1,117 @@
+/* sqrt function. PowerPC32 version.
+ Copyright (C) 2007 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, write to the Free
+ Software Foundation, Inc., 1 Franklin Street, Fifth Floor, Boston MA
+ 02110-1301 USA. */
+
+#include <sysdep.h>
+#include <math_ldbl_opt.h>
+
+/* double [fp1] sqrt (double x [fp1])
+ Power4 (ISA V2.0) and above implement sqrt in hardware (not optional).
+ The fsqrt instruction generates the correct value for all inputs and
+ sets the appropriate floating point exceptions. Extented checking is
+ only needed to set errno (via __kernel_standard) if the input value
+ is negative.
+
+ So compare the input value against the absolute value of itself.
+ This will compare equal unless the value is negative (EDOM) or a NAN,
+ in which case we branch to the extend wrapper. If equal we can return
+ the result directly.
+
+ This part of the function looks like a leaf routine, so no need to
+ stack a frame or execute prologue/epilogue code. It is safe to
+ branch directly to w_sqrt as long as the input value (f1) is
+ preserved. Putting the the sqrt result into f2 (float parameter 2)
+ allows passing both the input value and sqrt result into the extended
+ wrapper so there is no need to recompute.
+
+ This tactic avoids the overhead of stacking a frame for the normal
+ (non-error) case. Until gcc supports prologue shrink-wrapping
+ this is the best we can do. */
+
+ .section ".text"
+ .machine power4
+EALIGN (__sqrt, 5, 0)
+ fabs fp0,fp1
+ fsqrt fp2,fp1
+ fcmpu cr1,fp0,fp1
+ bne- cr1,.Lw_sqrt
+ fmr fp1,fp2
+ blr
+ .align 4
+.Lw_sqrt:
+ mflr r0
+ stwu r1,-16(r1)
+ cfi_adjust_cfa_offset(16)
+ fmr fp12,fp2
+ stw r0,20(r1)
+ stw r30,8(r1)
+ cfi_offset(lr,20)
+ cfi_offset(r30,8)
+#ifdef SHARED
+# ifdef HAVE_ASM_PPC_REL16
+ bcl 20,31,.LCF1
+.LCF1:
+ mflr r30
+ addis r30,r30,_GLOBAL_OFFSET_TABLE_-.LCF1@ha
+ addi r30,r30,_GLOBAL_OFFSET_TABLE_-.LCF1@l
+ lwz r9,_LIB_VERSION@got(30)
+ lwz r0,0(r9)
+# else
+ bl _GLOBAL_OFFSET_TABLE_@local-4
+ mflr r30
+ lwz r9,_LIB_VERSION@got(30)
+ lwz r0,0(r9)
+# endif
+#else
+ lis r9,_LIB_VERSION@ha
+ lwz r0,_LIB_VERSION@l(r9)
+#endif
+/* if (_LIB_VERSION == _IEEE_) return z; */
+ cmpwi cr7,r0,-1
+ beq- cr7,.L4
+/* if (x != x) return z; !isnan*/
+ fcmpu cr7,fp1,fp1
+ bne- cr7,.L4
+/* if (x < 0.0)
+ return __kernel_standard (x, x, 26) */
+ fmr fp2,fp1
+ li r3,26
+ bne- cr1,.L11
+.L4:
+ lwz r0,20(r1)
+ fmr fp1,fp12
+ lwz r30,8(r1)
+ addi r1,r1,16
+ mtlr r0
+ blr
+.L11:
+ bl __kernel_standard@plt
+ fmr fp12,fp1
+ b .L4
+ END (__sqrt)
+
+weak_alias (__sqrt, sqrt)
+
+#ifdef NO_LONG_DOUBLE
+weak_alias (__sqrt, sqrtl)
+strong_alias (__sqrt, __sqrtl)
+#endif
+#if LONG_DOUBLE_COMPAT(libm, GLIBC_2_0)
+compat_symbol (libm, __sqrt, sqrtl, GLIBC_2_0)
+#endif
+
diff -urN libc25-cvstip-20070919/sysdeps/powerpc/powerpc32/power5/fpu/w_sqrtf.S libc25/sysdeps/powerpc/powerpc32/power5/fpu/w_sqrtf.S
--- libc25-cvstip-20070919/sysdeps/powerpc/powerpc32/power5/fpu/w_sqrtf.S Wed Dec 31 18:00:00 1969
+++ libc25/sysdeps/powerpc/powerpc32/power5/fpu/w_sqrtf.S Wed Oct 31 12:24:11 2007
@@ -0,0 +1,109 @@
+/* sqrtf function. PowerPC32 version.
+ Copyright (C) 2007 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, write to the Free
+ Software Foundation, Inc., 1 Franklin Street, Fifth Floor, Boston MA
+ 02110-1301 USA. */
+
+#include <sysdep.h>
+#include <math_ldbl_opt.h>
+
+/* float [fp1] sqrts (float x [fp1])
+ Power4 (ISA V2.0) and above implement sqrt in hardware (not optional).
+ The fsqrts instruction generates the correct value for all inputs and
+ sets the appropriate floating point exceptions. Extented checking is
+ only needed to set errno (via __kernel_standard) if the input value
+ is negative.
+
+ So compare the input value against the absolute value of itself.
+ This will compare equal unless the value is negative (EDOM) or a NAN,
+ in which case we branch to the extend wrapper. If equal we can return
+ the result directly.
+
+ This part of the function looks like a leaf routine, so no need to
+ stack a frame or execute prologue/epilogue code. It is safe to
+ branch directly to w_sqrt as long as the input value (f1) is
+ preserved. Putting the the sqrt result into f2 (float parameter 2)
+ allows passing both the input value and sqrt result into the extended
+ wrapper so there is no need to recompute.
+
+ This tactic avoids the overhead of stacking a frame for the normal
+ (non-error) case. Until gcc supports prologue shrink-wrapping
+ this is the best we can do. */
+
+ .section ".text"
+ .machine power4
+EALIGN (__sqrtf, 5, 0)
+ fabs fp0,fp1
+ fsqrts fp2,fp1
+ fcmpu cr1,fp0,fp1
+ bne- cr1,.Lw_sqrtf
+ fmr fp1,fp2
+ blr
+ .align 4
+.Lw_sqrtf:
+ mflr r0
+ stwu r1,-16(r1)
+ cfi_adjust_cfa_offset(16)
+ fmr fp12,fp2
+ stw r0,20(r1)
+ stw r30,8(r1)
+ cfi_offset(lr,20)
+ cfi_offset(r30,8)
+#ifdef SHARED
+# ifdef HAVE_ASM_PPC_REL16
+ bcl 20,31,.LCF1
+.LCF1:
+ mflr r30
+ addis r30,r30,_GLOBAL_OFFSET_TABLE_-.LCF1@ha
+ addi r30,r30,_GLOBAL_OFFSET_TABLE_-.LCF1@l
+ lwz r9,_LIB_VERSION@got(30)
+ lwz r0,0(r9)
+# else
+ bl _GLOBAL_OFFSET_TABLE_@local-4
+ mflr r30
+ lwz r9,_LIB_VERSION@got(30)
+ lwz r0,0(r9)
+# endif
+#else
+ lis r9,_LIB_VERSION@ha
+ lwz r0,_LIB_VERSION@l(r9)
+#endif
+/* if (_LIB_VERSION == _IEEE_) return z; */
+ cmpwi cr7,r0,-1
+ beq- cr7,.L4
+/* if (x != x, 0) return z; !isnan */
+ fcmpu cr7,fp1,fp1
+ bne- cr7,.L4
+/* if (x < 0.0)
+ return __kernel_standard (x, x, 126) */
+ fmr fp2,fp1
+ li r3,126
+ bne- cr1,.L11
+.L4:
+ lwz r0,20(r1)
+ fmr fp1,fp12
+ lwz r30,8(r1)
+ addi r1,r1,16
+ mtlr r0
+ blr
+.L11:
+ bl __kernel_standard@plt
+ fmr fp12,fp1
+ b .L4
+ END (__sqrtf)
+
+weak_alias (__sqrtf, sqrtf)
+