This is the mail archive of the
glibc-cvs@sourceware.org
mailing list for the glibc project.
GNU C Library master sources branch master updated. glibc-2.26.9000-957-gf4b2aea
- From: tuliom at sourceware dot org
- To: glibc-cvs at sourceware dot org
- Date: 11 Dec 2017 19:47:50 -0000
- Subject: GNU C Library master sources branch master updated. glibc-2.26.9000-957-gf4b2aea
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "GNU C Library master sources".
The branch, master has been updated
via f4b2aea6e13345b73328fc5febf134d378e30554 (commit)
via c9cd7b0ce5c52a3dac7347084651d7df0b39a6d0 (commit)
from e70c6fee466d26af36a38270e451cbe3273a6660 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
http://sourceware.org/git/gitweb.cgi?p=glibc.git;a=commitdiff;h=f4b2aea6e13345b73328fc5febf134d378e30554
commit f4b2aea6e13345b73328fc5febf134d378e30554
Author: Paul Clarke <pc@us.ibm.com>
Date: Mon Dec 11 17:39:42 2017 -0200
New generic cosf
The same logic used in s_cosf.S version for x86 and powerpc
is used to create a generic s_cosf.c, so there is no performance
improvement in x86_64 and powerpc64.
* sysdeps/ieee754/flt-32/s_cosf.c: New implementation.
diff --git a/ChangeLog b/ChangeLog
index d5f7256..5ef7d5d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,7 @@
+2017-12-11 Paul A. Clarke <pc@us.ibm.com>
+
+ * sysdeps/ieee754/flt-32/s_cosf.c: New implementation.
+
2017-12-11 Adhemerval Zanella <azanella@linux.vnet.ibm.com>
Tulio Magno Quites Machado Filho <tuliom@linux.vnet.ibm.com>
diff --git a/sysdeps/ieee754/flt-32/s_cosf.c b/sysdeps/ieee754/flt-32/s_cosf.c
index 5ed0bca..ac6d044 100644
--- a/sysdeps/ieee754/flt-32/s_cosf.c
+++ b/sysdeps/ieee754/flt-32/s_cosf.c
@@ -1,21 +1,20 @@
-/* s_cosf.c -- float version of s_cos.c.
- * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com.
- */
-
-/*
- * ====================================================
- * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved.
- *
- * Developed at SunPro, a Sun Microsystems, Inc. business.
- * Permission to use, copy, modify, and distribute this
- * software is freely granted, provided that this notice
- * is preserved.
- * ====================================================
- */
-
-#if defined(LIBM_SCCS) && !defined(lint)
-static char rcsid[] = "$NetBSD: s_cosf.c,v 1.4 1995/05/10 20:47:03 jtc Exp $";
-#endif
+/* Compute cosine of argument.
+ Copyright (C) 2017 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
#include <errno.h>
#include <math.h>
@@ -28,35 +27,210 @@ static char rcsid[] = "$NetBSD: s_cosf.c,v 1.4 1995/05/10 20:47:03 jtc Exp $";
# define COSF_FUNC COSF
#endif
-float COSF_FUNC(float x)
+/* Chebyshev constants for cos, range -PI/4 - PI/4. */
+static const double C0 = -0x1.ffffffffe98aep-2;
+static const double C1 = 0x1.55555545c50c7p-5;
+static const double C2 = -0x1.6c16b348b6874p-10;
+static const double C3 = 0x1.a00eb9ac43ccp-16;
+static const double C4 = -0x1.23c97dd8844d7p-22;
+
+/* Chebyshev constants for sin, range -PI/4 - PI/4. */
+static const double S0 = -0x1.5555555551cd9p-3;
+static const double S1 = 0x1.1111110c2688bp-7;
+static const double S2 = -0x1.a019f8b4bd1f9p-13;
+static const double S3 = 0x1.71d7264e6b5b4p-19;
+static const double S4 = -0x1.a947e1674b58ap-26;
+
+/* Chebyshev constants for cos, range 2^-27 - 2^-5. */
+static const double CC0 = -0x1.fffffff5cc6fdp-2;
+static const double CC1 = 0x1.55514b178dac5p-5;
+
+/* PI/2 with 98 bits of accuracy. */
+static const double PI_2_hi = 0x1.921fb544p+0;
+static const double PI_2_lo = 0x1.0b4611a626332p-34;
+
+static const double inv_PI_4 = 0x1.45f306dc9c883p+0; /* 4/PI. */
+
+#define FLOAT_EXPONENT_SHIFT 23
+#define FLOAT_EXPONENT_BIAS 127
+
+static const double pio2_table[] = {
+ 0 * M_PI_2,
+ 1 * M_PI_2,
+ 2 * M_PI_2,
+ 3 * M_PI_2,
+ 4 * M_PI_2,
+ 5 * M_PI_2
+};
+
+static const double invpio4_table[] = {
+ 0x0p+0,
+ 0x1.45f306cp+0,
+ 0x1.c9c882ap-28,
+ 0x1.4fe13a8p-58,
+ 0x1.f47d4dp-85,
+ 0x1.bb81b6cp-112,
+ 0x1.4acc9ep-142,
+ 0x1.0e4107cp-169
+};
+
+static const double ones[] = { 1.0, -1.0 };
+
+/* Compute the cosine value using Chebyshev polynomials where
+ THETA is the range reduced absolute value of the input
+ and it is less than Pi/4,
+ N is calculated as trunc(|x|/(Pi/4)) + 1 and it is used to decide
+ whether a sine or cosine approximation is more accurate and
+ the sign of the result. */
+static inline float
+reduced (double theta, unsigned int n)
{
- float y[2],z=0.0;
- int32_t n,ix;
+ double sign, cx;
+ const double theta2 = theta * theta;
- GET_FLOAT_WORD(ix,x);
+ /* Determine positive or negative primary interval. */
+ n += 2;
+ sign = ones[(n >> 2) & 1];
- /* |x| ~< pi/4 */
- ix &= 0x7fffffff;
- if(ix <= 0x3f490fd8) return __kernel_cosf(x,z);
+ /* Are we in the primary interval of sin or cos? */
+ if ((n & 2) == 0)
+ {
+ /* Here cosf() is calculated using sin Chebyshev polynomial:
+ x+x^3*(S0+x^2*(S1+x^2*(S2+x^2*(S3+x^2*S4)))). */
+ cx = S3 + theta2 * S4;
+ cx = S2 + theta2 * cx;
+ cx = S1 + theta2 * cx;
+ cx = S0 + theta2 * cx;
+ cx = theta + theta * theta2 * cx;
+ }
+ else
+ {
+ /* Here cosf() is calculated using cos Chebyshev polynomial:
+ 1.0+x^2*(C0+x^2*(C1+x^2*(C2+x^2*(C3+x^2*C4)))). */
+ cx = C3 + theta2 * C4;
+ cx = C2 + theta2 * cx;
+ cx = C1 + theta2 * cx;
+ cx = C0 + theta2 * cx;
+ cx = 1. + theta2 * cx;
+ }
+ return sign * cx;
+}
- /* cos(Inf or NaN) is NaN */
- else if (ix>=0x7f800000) {
- if (ix == 0x7f800000)
- __set_errno (EDOM);
- return x-x;
+float
+COSF_FUNC (float x)
+{
+ double theta = x;
+ double abstheta = fabs (theta);
+ if (isless (abstheta, M_PI_4))
+ {
+ double cx;
+ if (abstheta >= 0x1p-5)
+ {
+ const double theta2 = theta * theta;
+ /* Chebyshev polynomial of the form for cos:
+ * 1 + x^2 (C0 + x^2 (C1 + x^2 (C2 + x^2 (C3 + x^2 * C4)))). */
+ cx = C3 + theta2 * C4;
+ cx = C2 + theta2 * cx;
+ cx = C1 + theta2 * cx;
+ cx = C0 + theta2 * cx;
+ cx = 1. + theta2 * cx;
+ return cx;
}
-
- /* argument reduction needed */
- else {
- n = __ieee754_rem_pio2f(x,y);
- switch(n&3) {
- case 0: return __kernel_cosf(y[0],y[1]);
- case 1: return -__kernel_sinf(y[0],y[1],1);
- case 2: return -__kernel_cosf(y[0],y[1]);
- default:
- return __kernel_sinf(y[0],y[1],1);
+ else if (abstheta >= 0x1p-27)
+ {
+ /* A simpler Chebyshev approximation is close enough for this range:
+ * 1 + x^2 (CC0 + x^3 * CC1). */
+ const double theta2 = theta * theta;
+ cx = CC0 + theta * theta2 * CC1;
+ cx = 1.0 + theta2 * cx;
+ return cx;
+ }
+ else
+ {
+ /* For small enough |theta|, this is close enough. */
+ return 1.0 - abstheta;
+ }
+ }
+ else /* |theta| >= Pi/4. */
+ {
+ if (isless (abstheta, 9 * M_PI_4))
+ {
+ /* There are cases where FE_UPWARD rounding mode can
+ produce a result of abstheta * inv_PI_4 == 9,
+ where abstheta < 9pi/4, so the domain for
+ pio2_table must go to 5 (9 / 2 + 1). */
+ unsigned int n = (abstheta * inv_PI_4) + 1;
+ theta = abstheta - pio2_table[n / 2];
+ return reduced (theta, n);
+ }
+ else if (isless (abstheta, INFINITY))
+ {
+ if (abstheta < 0x1p+23)
+ {
+ unsigned int n = ((unsigned int) (abstheta * inv_PI_4)) + 1;
+ double x = n / 2;
+ theta = (abstheta - x * PI_2_hi) - x * PI_2_lo;
+ /* Argument reduction needed. */
+ return reduced (theta, n);
+ }
+ else /* |theta| >= 2^23. */
+ {
+ x = fabsf (x);
+ int exponent;
+ GET_FLOAT_WORD (exponent, x);
+ exponent = (exponent >> FLOAT_EXPONENT_SHIFT)
+ - FLOAT_EXPONENT_BIAS;
+ exponent += 3;
+ exponent /= 28;
+ double a = invpio4_table[exponent] * x;
+ double b = invpio4_table[exponent + 1] * x;
+ double c = invpio4_table[exponent + 2] * x;
+ double d = invpio4_table[exponent + 3] * x;
+ uint64_t l = a;
+ l &= ~0x7;
+ a -= l;
+ double e = a + b;
+ l = e;
+ e = a - l;
+ if (l & 1)
+ {
+ e -= 1.0;
+ e += b;
+ e += c;
+ e += d;
+ e *= M_PI_4;
+ return reduced (e, l + 1);
+ }
+ else
+ {
+ e += b;
+ e += c;
+ e += d;
+ if (e <= 1.0)
+ {
+ e *= M_PI_4;
+ return reduced (e, l + 1);
+ }
+ else
+ {
+ l++;
+ e -= 2.0;
+ e *= M_PI_4;
+ return reduced (e, l + 1);
+ }
+ }
}
}
+ else
+ {
+ int32_t ix;
+ GET_FLOAT_WORD (ix, abstheta);
+ /* cos(Inf or NaN) is NaN. */
+ if (ix == 0x7f800000) /* Inf. */
+ __set_errno (EDOM);
+ return x - x;
+ }
+ }
}
#ifndef COSF
http://sourceware.org/git/gitweb.cgi?p=glibc.git;a=commitdiff;h=c9cd7b0ce5c52a3dac7347084651d7df0b39a6d0
commit c9cd7b0ce5c52a3dac7347084651d7df0b39a6d0
Author: Adhemerval Zanella <azanella@linux.vnet.ibm.com>
Date: Mon Dec 11 17:39:42 2017 -0200
powerpc: POWER8 memcpy optimization for cached memory
On POWER8, unaligned memory accesses to cached memory has little impact
on performance as opposed to its ancestors.
It is disabled by default and will only be available when the tunable
glibc.tune.cached_memopt is set to 1.
__memcpy_power8_cached __memcpy_power7
============================================================
max-size=4096: 33325.70 ( 12.65%) 38153.00
max-size=8192: 32878.20 ( 11.17%) 37012.30
max-size=16384: 33782.20 ( 11.61%) 38219.20
max-size=32768: 33296.20 ( 11.30%) 37538.30
max-size=65536: 33765.60 ( 10.53%) 37738.40
* manual/tunables.texi (Hardware Capability Tunables): Document
glibc.tune.cached_memopt.
* sysdeps/powerpc/cpu-features.c: New file.
* sysdeps/powerpc/cpu-features.h: New file.
* sysdeps/powerpc/dl-procinfo.c [!IS_IN(ldconfig)]: Add
_dl_powerpc_cpu_features.
* sysdeps/powerpc/dl-tunables.list: New file.
* sysdeps/powerpc/ldsodefs.h: Include cpu-features.h.
* sysdeps/powerpc/powerpc32/power4/multiarch/init-arch.h
(INIT_ARCH): Initialize use_aligned_memopt.
* sysdeps/powerpc/powerpc64/dl-machine.h [defined(SHARED &&
IS_IN(rtld))]: Restrict dl_platform_init availability and
initialize CPU features used by tunables.
* sysdeps/powerpc/powerpc64/multiarch/Makefile (sysdep_routines):
Add memcpy-power8-cached.
* sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c: Add
__memcpy_power8_cached.
* sysdeps/powerpc/powerpc64/multiarch/memcpy.c: Likewise.
* sysdeps/powerpc/powerpc64/multiarch/memcpy-power8-cached.S:
New file.
Reviewed-by: Rajalakshmi Srinivasaraghavan <raji@linux.vnet.ibm.com>
diff --git a/ChangeLog b/ChangeLog
index 50da031..d5f7256 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,27 @@
+2017-12-11 Adhemerval Zanella <azanella@linux.vnet.ibm.com>
+ Tulio Magno Quites Machado Filho <tuliom@linux.vnet.ibm.com>
+
+ * manual/tunables.texi (Hardware Capability Tunables): Document
+ glibc.tune.cached_memopt.
+ * sysdeps/powerpc/cpu-features.c: New file.
+ * sysdeps/powerpc/cpu-features.h: New file.
+ * sysdeps/powerpc/dl-procinfo.c [!IS_IN(ldconfig)]: Add
+ _dl_powerpc_cpu_features.
+ * sysdeps/powerpc/dl-tunables.list: New file.
+ * sysdeps/powerpc/ldsodefs.h: Include cpu-features.h.
+ * sysdeps/powerpc/powerpc32/power4/multiarch/init-arch.h
+ (INIT_ARCH): Initialize use_aligned_memopt.
+ * sysdeps/powerpc/powerpc64/dl-machine.h [defined(SHARED &&
+ IS_IN(rtld))]: Restrict dl_platform_init availability and
+ initialize CPU features used by tunables.
+ * sysdeps/powerpc/powerpc64/multiarch/Makefile (sysdep_routines):
+ Add memcpy-power8-cached.
+ * sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c: Add
+ __memcpy_power8_cached.
+ * sysdeps/powerpc/powerpc64/multiarch/memcpy.c: Likewise.
+ * sysdeps/powerpc/powerpc64/multiarch/memcpy-power8-cached.S:
+ New file.
+
2017-12-11 H.J. Lu <hongjiu.lu@intel.com>
* string/Makefile (CFLAGS-inl-tester.c): Replace = with +=.
diff --git a/manual/tunables.texi b/manual/tunables.texi
index e851b95..6e0ee28 100644
--- a/manual/tunables.texi
+++ b/manual/tunables.texi
@@ -319,6 +319,16 @@ the ones in @code{sysdeps/x86/cpu-features.h}.
This tunable is specific to i386 and x86-64.
@end deftp
+@deftp Tunable glibc.tune.cached_memopt
+The @code{glibc.tune.cached_memopt=[0|1]} tunable allows the user to
+enable optimizations recommended for cacheable memory. If set to
+@code{1}, @theglibc{} assumes that the process memory image consists
+of cacheable (non-device) memory only. The default, @code{0},
+indicates that the process may use device memory.
+
+This tunable is specific to powerpc, powerpc64 and powerpc64le.
+@end deftp
+
@deftp Tunable glibc.tune.cpu
The @code{glibc.tune.cpu=xxx} tunable allows the user to tell @theglibc{} to
assume that the CPU is @code{xxx} where xxx may have one of these values:
diff --git a/sysdeps/powerpc/cpu-features.c b/sysdeps/powerpc/cpu-features.c
new file mode 100644
index 0000000..6870582
--- /dev/null
+++ b/sysdeps/powerpc/cpu-features.c
@@ -0,0 +1,39 @@
+/* Initialize cpu feature data. PowerPC version.
+ Copyright (C) 2017 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#include <stdint.h>
+#include <cpu-features.h>
+
+#if HAVE_TUNABLES
+# include <elf/dl-tunables.h>
+#endif
+
+static inline void
+init_cpu_features (struct cpu_features *cpu_features)
+{
+ /* Default is to use aligned memory access on optimized function unless
+ tunables is enable, since for this case user can explicit disable
+ unaligned optimizations. */
+#if HAVE_TUNABLES
+ int32_t cached_memfunc = TUNABLE_GET (glibc, tune, cached_memopt, int32_t,
+ NULL);
+ cpu_features->use_cached_memopt = (cached_memfunc > 0);
+#else
+ cpu_features->use_cached_memopt = false;
+#endif
+}
diff --git a/sysdeps/powerpc/cpu-features.h b/sysdeps/powerpc/cpu-features.h
new file mode 100644
index 0000000..36a8bb4
--- /dev/null
+++ b/sysdeps/powerpc/cpu-features.h
@@ -0,0 +1,28 @@
+/* Initialize cpu feature data. PowerPC version.
+ Copyright (C) 2017 Free Software Foundation, Inc.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#ifndef __CPU_FEATURES_POWERPC_H
+# define __CPU_FEATURES_POWERPC_H
+
+#include <stdbool.h>
+
+struct cpu_features
+{
+ bool use_cached_memopt;
+};
+
+#endif /* __CPU_FEATURES_H */
diff --git a/sysdeps/powerpc/dl-procinfo.c b/sysdeps/powerpc/dl-procinfo.c
index 55a6e78..c8b1445 100644
--- a/sysdeps/powerpc/dl-procinfo.c
+++ b/sysdeps/powerpc/dl-procinfo.c
@@ -42,6 +42,22 @@
# define PROCINFO_CLASS
#endif
+#if !IS_IN (ldconfig)
+# if !defined PROCINFO_DECL && defined SHARED
+ ._dl_powerpc_cpu_features
+# else
+PROCINFO_CLASS struct cpu_features _dl_powerpc_cpu_features
+# endif
+# ifndef PROCINFO_DECL
+= { }
+# endif
+# if !defined SHARED || defined PROCINFO_DECL
+;
+# else
+,
+# endif
+#endif
+
#if !defined PROCINFO_DECL && defined SHARED
._dl_powerpc_cap_flags
#else
diff --git a/sysdeps/powerpc/dl-tunables.list b/sysdeps/powerpc/dl-tunables.list
new file mode 100644
index 0000000..9e14b9a
--- /dev/null
+++ b/sysdeps/powerpc/dl-tunables.list
@@ -0,0 +1,28 @@
+# powerpc specific tunables.
+# Copyright (C) 2017 Free Software Foundation, Inc.
+# This file is part of the GNU C Library.
+
+# The GNU C Library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+
+# The GNU C Library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with the GNU C Library; if not, see
+# <http://www.gnu.org/licenses/>.
+
+glibc {
+ tune {
+ cached_memopt {
+ type: INT_32
+ minval: 0
+ maxval: 1
+ default: 0
+ }
+ }
+}
diff --git a/sysdeps/powerpc/ldsodefs.h b/sysdeps/powerpc/ldsodefs.h
index 466de79..6f8b3a2 100644
--- a/sysdeps/powerpc/ldsodefs.h
+++ b/sysdeps/powerpc/ldsodefs.h
@@ -20,6 +20,7 @@
#define _POWERPC_LDSODEFS_H 1
#include <elf.h>
+#include <cpu-features.h>
struct La_ppc32_regs;
struct La_ppc32_retval;
diff --git a/sysdeps/powerpc/powerpc32/power4/multiarch/init-arch.h b/sysdeps/powerpc/powerpc32/power4/multiarch/init-arch.h
index f2e6a4b..6038941 100644
--- a/sysdeps/powerpc/powerpc32/power4/multiarch/init-arch.h
+++ b/sysdeps/powerpc/powerpc32/power4/multiarch/init-arch.h
@@ -37,6 +37,8 @@
#define INIT_ARCH() \
unsigned long int hwcap = __GLRO(dl_hwcap); \
unsigned long int __attribute__((unused)) hwcap2 = __GLRO(dl_hwcap2); \
+ bool __attribute__((unused)) use_cached_memopt = \
+ GLRO(dl_powerpc_cpu_features).use_cached_memopt; \
if (hwcap & PPC_FEATURE_ARCH_2_06) \
hwcap |= PPC_FEATURE_ARCH_2_05 | \
PPC_FEATURE_POWER5_PLUS | \
diff --git a/sysdeps/powerpc/powerpc64/dl-machine.h b/sysdeps/powerpc/powerpc64/dl-machine.h
index aeb91b8..76dceee 100644
--- a/sysdeps/powerpc/powerpc64/dl-machine.h
+++ b/sysdeps/powerpc/powerpc64/dl-machine.h
@@ -27,6 +27,7 @@
#include <dl-tls.h>
#include <sysdep.h>
#include <hwcapinfo.h>
+#include <cpu-features.c>
/* Translate a processor specific dynamic tag to the index
in l_info array. */
@@ -300,13 +301,14 @@ BODY_PREFIX "_dl_start_user:\n" \
/* We define an initialization function to initialize HWCAP/HWCAP2 and
platform data so it can be copied into the TCB later. This is called
very early in _dl_sysdep_start for dynamically linked binaries. */
-#ifdef SHARED
+#if defined(SHARED) && IS_IN (rtld)
# define DL_PLATFORM_INIT dl_platform_init ()
static inline void __attribute__ ((unused))
dl_platform_init (void)
{
__tcb_parse_hwcap_and_convert_at_platform ();
+ init_cpu_features (&GLRO(dl_powerpc_cpu_features));
}
#endif
diff --git a/sysdeps/powerpc/powerpc64/multiarch/Makefile b/sysdeps/powerpc/powerpc64/multiarch/Makefile
index dea49ac..4df6b45 100644
--- a/sysdeps/powerpc/powerpc64/multiarch/Makefile
+++ b/sysdeps/powerpc/powerpc64/multiarch/Makefile
@@ -1,6 +1,6 @@
ifeq ($(subdir),string)
-sysdep_routines += memcpy-power7 memcpy-a2 memcpy-power6 memcpy-cell \
- memcpy-power4 memcpy-ppc64 \
+sysdep_routines += memcpy-power8-cached memcpy-power7 memcpy-a2 memcpy-power6 \
+ memcpy-cell memcpy-power4 memcpy-ppc64 \
memcmp-power8 memcmp-power7 memcmp-power4 memcmp-ppc64 \
memset-power7 memset-power6 memset-power4 \
memset-ppc64 memset-power8 \
diff --git a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
index 6a88536..77a60ea 100644
--- a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
@@ -51,6 +51,8 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
#ifdef SHARED
/* Support sysdeps/powerpc/powerpc64/multiarch/memcpy.c. */
IFUNC_IMPL (i, name, memcpy,
+ IFUNC_IMPL_ADD (array, i, memcpy, hwcap2 & PPC_FEATURE2_ARCH_2_07,
+ __memcpy_power8_cached)
IFUNC_IMPL_ADD (array, i, memcpy, hwcap & PPC_FEATURE_HAS_VSX,
__memcpy_power7)
IFUNC_IMPL_ADD (array, i, memcpy, hwcap & PPC_FEATURE_ARCH_2_06,
diff --git a/sysdeps/powerpc/powerpc64/multiarch/memcpy-power8-cached.S b/sysdeps/powerpc/powerpc64/multiarch/memcpy-power8-cached.S
new file mode 100644
index 0000000..e8bea91
--- /dev/null
+++ b/sysdeps/powerpc/powerpc64/multiarch/memcpy-power8-cached.S
@@ -0,0 +1,176 @@
+/* Optimized memcpy implementation for cached memory on PowerPC64/POWER8.
+ Copyright (C) 2017 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#include <sysdep.h>
+
+
+/* __ptr_t [r3] memcpy (__ptr_t dst [r3], __ptr_t src [r4], size_t len [r5]);
+ Returns 'dst'. */
+
+ .machine power8
+ENTRY_TOCLESS (__memcpy_power8_cached, 5)
+ CALL_MCOUNT 3
+
+ cmpldi cr7,r5,15
+ bgt cr7,L(ge_16)
+ andi. r9,r5,0x1
+ mr r9,r3
+ beq cr0,1f
+ lbz r10,0(r4)
+ addi r9,r3,1
+ addi r4,r4,1
+ stb r10,0(r3)
+1:
+ andi. r10,r5,0x2
+ beq cr0,2f
+ lhz r10,0(r4)
+ addi r9,r9,2
+ addi r4,r4,2
+ sth r10,-2(r9)
+2:
+ andi. r10,r5,0x4
+ beq cr0,3f
+ lwz r10,0(r4)
+ addi r9,9,4
+ addi r4,4,4
+ stw r10,-4(r9)
+3:
+ andi. r10,r5,0x8
+ beqlr cr0
+ ld r10,0(r4)
+ std r10,0(r9)
+ blr
+
+ .align 4
+L(ge_16):
+ cmpldi cr7,r5,32
+ ble cr7,L(ge_16_le_32)
+ cmpldi cr7,r5,64
+ ble cr7,L(gt_32_le_64)
+
+ /* Align dst to 16 bytes. */
+ andi. r9,r3,0xf
+ mr r12,r3
+ beq cr0,L(dst_is_align_16)
+ lxvd2x v0,0,r4
+ subfic r12,r9,16
+ subf r5,r12,r5
+ add r4,r4,r12
+ add r12,r3,r12
+ stxvd2x v0,0,r3
+L(dst_is_align_16):
+ cmpldi cr7,r5,127
+ ble cr7,L(tail_copy)
+ mr r9,r12
+ srdi r10,r5,7
+ li r11,16
+ li r6,32
+ li r7,48
+ mtctr r10
+ clrrdi r0,r5,7
+
+ /* Main loop, copy 128 bytes each time. */
+ .align 4
+L(copy_128):
+ lxvd2x v10,0,r4
+ lxvd2x v11,r4,r11
+ addi r8,r4,64
+ addi r10,r9,64
+ lxvd2x v12,r4,r6
+ lxvd2x v0,r4,r7
+ addi r4,r4,128
+ stxvd2x v10,0,r9
+ stxvd2x v11,r9,r11
+ stxvd2x v12,r9,r6
+ stxvd2x v0,r9,r7
+ addi r9,r9,128
+ lxvd2x v10,0,r8
+ lxvd2x v11,r8,r11
+ lxvd2x v12,r8,r6
+ lxvd2x v0,r8,r7
+ stxvd2x v10,0,r10
+ stxvd2x v11,r10,r11
+ stxvd2x v12,r10,r6
+ stxvd2x v0,r10,r7
+ bdnz L(copy_128)
+
+ add r12,r12,r0
+ rldicl r5,r5,0,57
+L(tail_copy):
+ cmpldi cr7,r5,63
+ ble cr7,L(tail_le_64)
+ li r8,16
+ li r10,32
+ lxvd2x v10,0,r4
+ li r9,48
+ addi r5,r5,-64
+ lxvd2x v11,r4,r8
+ lxvd2x v12,r4,r10
+ lxvd2x v0,r4,r9
+ addi r4,r4,64
+ stxvd2x v10,0,r12
+ stxvd2x v11,r12,r8
+ stxvd2x v12,r12,r10
+ stxvd2x v0,r12,9
+ addi r12,r12,64
+
+L(tail_le_64):
+ cmpldi cr7,r5,32
+ bgt cr7,L(tail_gt_32_le_64)
+ cmpdi cr7,r5,0
+ beqlr cr7
+ addi r5,r5,-32
+ li r9,16
+ add r8,r4,r5
+ add r10,r12,r5
+ lxvd2x v12,r4,r5
+ lxvd2x v0,r8,r9
+ stxvd2x v12,r12,r5
+ stxvd2x v0,r10,r9
+ blr
+
+ .align 4
+L(ge_16_le_32):
+ addi r5,r5,-16
+ lxvd2x v0,0,r4
+ lxvd2x v1,r4,r5
+ stxvd2x v0,0,r3
+ stxvd2x v1,r3,r5
+ blr
+
+ .align 4
+L(gt_32_le_64):
+ mr r12,r3
+
+ .align 4
+L(tail_gt_32_le_64):
+ li r9,16
+ lxvd2x v0,0,r4
+ addi r5,r5,-32
+ lxvd2x v1,r4,r9
+ add r8,r4,r5
+ lxvd2x v2,r4,r5
+ add r10,r12,r5
+ lxvd2x v3,r8,r9
+ stxvd2x v0,0,r12
+ stxvd2x v1,r12,r9
+ stxvd2x v2,r12,r5
+ stxvd2x v3,r10,r9
+ blr
+
+END_GEN_TB (__memcpy_power8_cached,TB_TOCLESS)
diff --git a/sysdeps/powerpc/powerpc64/multiarch/memcpy.c b/sysdeps/powerpc/powerpc64/multiarch/memcpy.c
index 9f4286c..fb49fe1 100644
--- a/sysdeps/powerpc/powerpc64/multiarch/memcpy.c
+++ b/sysdeps/powerpc/powerpc64/multiarch/memcpy.c
@@ -35,18 +35,21 @@ extern __typeof (__redirect_memcpy) __memcpy_cell attribute_hidden;
extern __typeof (__redirect_memcpy) __memcpy_power6 attribute_hidden;
extern __typeof (__redirect_memcpy) __memcpy_a2 attribute_hidden;
extern __typeof (__redirect_memcpy) __memcpy_power7 attribute_hidden;
+extern __typeof (__redirect_memcpy) __memcpy_power8_cached attribute_hidden;
libc_ifunc (__libc_memcpy,
- (hwcap & PPC_FEATURE_HAS_VSX)
- ? __memcpy_power7 :
- (hwcap & PPC_FEATURE_ARCH_2_06)
- ? __memcpy_a2 :
- (hwcap & PPC_FEATURE_ARCH_2_05)
- ? __memcpy_power6 :
- (hwcap & PPC_FEATURE_CELL_BE)
- ? __memcpy_cell :
- (hwcap & PPC_FEATURE_POWER4)
- ? __memcpy_power4
+ ((hwcap2 & PPC_FEATURE2_ARCH_2_07) && use_cached_memopt)
+ ? __memcpy_power8_cached :
+ (hwcap & PPC_FEATURE_HAS_VSX)
+ ? __memcpy_power7 :
+ (hwcap & PPC_FEATURE_ARCH_2_06)
+ ? __memcpy_a2 :
+ (hwcap & PPC_FEATURE_ARCH_2_05)
+ ? __memcpy_power6 :
+ (hwcap & PPC_FEATURE_CELL_BE)
+ ? __memcpy_cell :
+ (hwcap & PPC_FEATURE_POWER4)
+ ? __memcpy_power4
: __memcpy_ppc);
#undef memcpy
-----------------------------------------------------------------------
Summary of changes:
ChangeLog | 28 +++
manual/tunables.texi | 10 +
sysdeps/ieee754/flt-32/s_cosf.c | 254 ++++++++++++++++---
sysdeps/powerpc/cpu-features.c | 39 +++
sysdeps/powerpc/cpu-features.h | 28 +++
sysdeps/powerpc/dl-procinfo.c | 16 ++
sysdeps/powerpc/dl-tunables.list | 28 +++
sysdeps/powerpc/ldsodefs.h | 1 +
.../powerpc/powerpc32/power4/multiarch/init-arch.h | 2 +
sysdeps/powerpc/powerpc64/dl-machine.h | 4 +-
sysdeps/powerpc/powerpc64/multiarch/Makefile | 4 +-
.../powerpc/powerpc64/multiarch/ifunc-impl-list.c | 2 +
.../powerpc64/multiarch/memcpy-power8-cached.S | 176 ++++++++++++++
sysdeps/powerpc/powerpc64/multiarch/memcpy.c | 23 +-
14 files changed, 562 insertions(+), 53 deletions(-)
create mode 100644 sysdeps/powerpc/cpu-features.c
create mode 100644 sysdeps/powerpc/cpu-features.h
create mode 100644 sysdeps/powerpc/dl-tunables.list
create mode 100644 sysdeps/powerpc/powerpc64/multiarch/memcpy-power8-cached.S
hooks/post-receive
--
GNU C Library master sources