This is the mail archive of the
glibc-cvs@sourceware.org
mailing list for the glibc project.
GNU C Library master sources branch hjl/pr21258/2.23 created. glibc-2.23-97-g883cadc
- From: hjl at sourceware dot org
- To: glibc-cvs at sourceware dot org
- Date: 20 Apr 2017 14:58:07 -0000
- Subject: GNU C Library master sources branch hjl/pr21258/2.23 created. glibc-2.23-97-g883cadc
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "GNU C Library master sources".
The branch, hjl/pr21258/2.23 has been created
at 883cadc5543ffd3a4537498b44c782ded8a4a4e8 (commit)
- Log -----------------------------------------------------------------
http://sourceware.org/git/gitweb.cgi?p=glibc.git;a=commitdiff;h=883cadc5543ffd3a4537498b44c782ded8a4a4e8
commit 883cadc5543ffd3a4537498b44c782ded8a4a4e8
Author: H.J. Lu <hjl.tools@gmail.com>
Date: Tue Mar 21 10:59:31 2017 -0700
x86-64: Improve branch predication in _dl_runtime_resolve_avx512_opt [BZ #21258]
On Skylake server, _dl_runtime_resolve_avx512_opt is used to preserve
the first 8 vector registers. The code layout is
if only %xmm0 - %xmm7 registers are used
preserve %xmm0 - %xmm7 registers
if only %ymm0 - %ymm7 registers are used
preserve %ymm0 - %ymm7 registers
preserve %zmm0 - %zmm7 registers
Branch predication always executes the fallthrough code path to preserve
%zmm0 - %zmm7 registers speculatively, even though only %xmm0 - %xmm7
registers are used. This leads to lower CPU frequency on Skylake
server. This patch changes the fallthrough code path to preserve
%xmm0 - %xmm7 registers instead:
if whole %zmm0 - %zmm7 registers are used
preserve %zmm0 - %zmm7 registers
if only %ymm0 - %ymm7 registers are used
preserve %ymm0 - %ymm7 registers
preserve %xmm0 - %xmm7 registers
Tested on Skylake server.
[BZ #21258]
* sysdeps/x86_64/dl-trampoline.S (_dl_runtime_resolve_opt):
Define only if _dl_runtime_resolve is defined to
_dl_runtime_resolve_sse_vex.
* sysdeps/x86_64/dl-trampoline.h (_dl_runtime_resolve_opt):
Fallthrough to _dl_runtime_resolve_sse_vex.
(cherry picked from commit c15f8eb50cea7ad1a4ccece6e0982bf426d52c00)
diff --git a/sysdeps/x86_64/dl-trampoline.S b/sysdeps/x86_64/dl-trampoline.S
index 3f812b8..177f666 100644
--- a/sysdeps/x86_64/dl-trampoline.S
+++ b/sysdeps/x86_64/dl-trampoline.S
@@ -88,11 +88,9 @@
# endif
# define VEC(i) zmm##i
# define _dl_runtime_resolve _dl_runtime_resolve_avx512
-# define _dl_runtime_resolve_opt _dl_runtime_resolve_avx512_opt
# define _dl_runtime_profile _dl_runtime_profile_avx512
# include "dl-trampoline.h"
# undef _dl_runtime_resolve
-# undef _dl_runtime_resolve_opt
# undef _dl_runtime_profile
# undef VEC
# undef VMOV
@@ -101,8 +99,6 @@
#else
strong_alias (_dl_runtime_resolve_avx, _dl_runtime_resolve_avx512)
.hidden _dl_runtime_resolve_avx512
-strong_alias (_dl_runtime_resolve_avx_opt, _dl_runtime_resolve_avx512_opt)
- .hidden _dl_runtime_resolve_avx512_opt
strong_alias (_dl_runtime_profile_avx, _dl_runtime_profile_avx512)
.hidden _dl_runtime_profile_avx512
#endif
@@ -154,4 +150,5 @@ strong_alias (_dl_runtime_profile_avx, _dl_runtime_profile_avx512)
# define VMOV vmovdqu
#endif
#define _dl_runtime_resolve _dl_runtime_resolve_sse_vex
+#define _dl_runtime_resolve_opt _dl_runtime_resolve_avx512_opt
#include "dl-trampoline.h"
diff --git a/sysdeps/x86_64/dl-trampoline.h b/sysdeps/x86_64/dl-trampoline.h
index abe4471..32ad3af 100644
--- a/sysdeps/x86_64/dl-trampoline.h
+++ b/sysdeps/x86_64/dl-trampoline.h
@@ -129,19 +129,20 @@ _dl_runtime_resolve_opt:
# YMM state isn't in use.
PRESERVE_BND_REGS_PREFIX
jz _dl_runtime_resolve_sse_vex
-# elif VEC_SIZE == 64
+# elif VEC_SIZE == 16
# For ZMM registers, check if YMM state and ZMM state are in
# use.
andl $(bit_YMM_state | bit_ZMM0_15_state), %r11d
cmpl $bit_YMM_state, %r11d
- # Preserve %xmm0 - %xmm7 registers with the zero upper 384 bits if
- # neither YMM state nor ZMM state are in use.
+ # Preserve %zmm0 - %zmm7 registers if ZMM state is in use.
PRESERVE_BND_REGS_PREFIX
- jl _dl_runtime_resolve_sse_vex
+ jg _dl_runtime_resolve_avx512
# Preserve %ymm0 - %ymm7 registers with the zero upper 256 bits if
# ZMM state isn't in use.
PRESERVE_BND_REGS_PREFIX
je _dl_runtime_resolve_avx
+ # Preserve %xmm0 - %xmm7 registers with the zero upper 384 bits if
+ # neither YMM state nor ZMM state are in use.
# else
# error Unsupported VEC_SIZE!
# endif
http://sourceware.org/git/gitweb.cgi?p=glibc.git;a=commitdiff;h=83037ea1d9e84b1b44ed307f01cbb5eeac24e22d
commit 83037ea1d9e84b1b44ed307f01cbb5eeac24e22d
Author: H.J. Lu <hjl.tools@gmail.com>
Date: Tue Aug 23 09:09:32 2016 -0700
X86-64: Add _dl_runtime_resolve_avx[512]_{opt|slow} [BZ #20508]
There is transition penalty when SSE instructions are mixed with 256-bit
AVX or 512-bit AVX512 load instructions. Since _dl_runtime_resolve_avx
and _dl_runtime_profile_avx512 save/restore 256-bit YMM/512-bit ZMM
registers, there is transition penalty when SSE instructions are used
with lazy binding on AVX and AVX512 processors.
To avoid SSE transition penalty, if only the lower 128 bits of the first
8 vector registers are non-zero, we can preserve %xmm0 - %xmm7 registers
with the zero upper bits.
For AVX and AVX512 processors which support XGETBV with ECX == 1, we can
use XGETBV with ECX == 1 to check if the upper 128 bits of YMM registers
or the upper 256 bits of ZMM registers are zero. We can restore only the
non-zero portion of vector registers with AVX/AVX512 load instructions
which will zero-extend upper bits of vector registers.
This patch adds _dl_runtime_resolve_sse_vex which saves and restores
XMM registers with 128-bit AVX store/load instructions. It is used to
preserve YMM/ZMM registers when only the lower 128 bits are non-zero.
_dl_runtime_resolve_avx_opt and _dl_runtime_resolve_avx512_opt are added
and used on AVX/AVX512 processors supporting XGETBV with ECX == 1 so
that we store and load only the non-zero portion of vector registers.
This avoids SSE transition penalty caused by _dl_runtime_resolve_avx and
_dl_runtime_profile_avx512 when only the lower 128 bits of vector
registers are used.
_dl_runtime_resolve_avx_slow is added and used for AVX processors which
don't support XGETBV with ECX == 1. Since there is no SSE transition
penalty on AVX512 processors which don't support XGETBV with ECX == 1,
_dl_runtime_resolve_avx512_slow isn't provided.
[BZ #20495]
[BZ #20508]
* sysdeps/x86/cpu-features.c (init_cpu_features): For Intel
processors, set Use_dl_runtime_resolve_slow and set
Use_dl_runtime_resolve_opt if XGETBV suports ECX == 1.
* sysdeps/x86/cpu-features.h (bit_Use_dl_runtime_resolve_opt):
New.
(bit_Use_dl_runtime_resolve_slow): Likewise.
(index_Use_dl_runtime_resolve_opt): Likewise.
(index_Use_dl_runtime_resolve_slow): Likewise.
* sysdeps/x86_64/dl-machine.h (elf_machine_runtime_setup): Use
_dl_runtime_resolve_avx512_opt and _dl_runtime_resolve_avx_opt
if Use_dl_runtime_resolve_opt is set. Use
_dl_runtime_resolve_slow if Use_dl_runtime_resolve_slow is set.
* sysdeps/x86_64/dl-trampoline.S: Include <cpu-features.h>.
(_dl_runtime_resolve_opt): New. Defined for AVX and AVX512.
(_dl_runtime_resolve): Add one for _dl_runtime_resolve_sse_vex.
* sysdeps/x86_64/dl-trampoline.h (_dl_runtime_resolve_avx_slow):
New.
(_dl_runtime_resolve_opt): Likewise.
(_dl_runtime_profile): Define only if _dl_runtime_profile is
defined.
(cherry picked from commit fb0f7a6755c1bfaec38f490fbfcaa39a66ee3604)
diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c
index 218ff2b..b1915cd 100644
--- a/sysdeps/x86/cpu-features.c
+++ b/sysdeps/x86/cpu-features.c
@@ -134,6 +134,20 @@ init_cpu_features (struct cpu_features *cpu_features)
break;
}
}
+
+ /* To avoid SSE transition penalty, use _dl_runtime_resolve_slow.
+ If XGETBV suports ECX == 1, use _dl_runtime_resolve_opt. */
+ cpu_features->feature[index_Use_dl_runtime_resolve_slow]
+ |= bit_Use_dl_runtime_resolve_slow;
+ if (cpu_features->max_cpuid >= 0xd)
+ {
+ unsigned int eax;
+
+ __cpuid_count (0xd, 1, eax, ebx, ecx, edx);
+ if ((eax & (1 << 2)) != 0)
+ cpu_features->feature[index_Use_dl_runtime_resolve_opt]
+ |= bit_Use_dl_runtime_resolve_opt;
+ }
}
/* This spells out "AuthenticAMD". */
else if (ebx == 0x68747541 && ecx == 0x444d4163 && edx == 0x69746e65)
diff --git a/sysdeps/x86/cpu-features.h b/sysdeps/x86/cpu-features.h
index e354920..22c9eaf 100644
--- a/sysdeps/x86/cpu-features.h
+++ b/sysdeps/x86/cpu-features.h
@@ -35,6 +35,8 @@
#define bit_I686 (1 << 15)
#define bit_Prefer_MAP_32BIT_EXEC (1 << 16)
#define bit_Prefer_No_VZEROUPPER (1 << 17)
+#define bit_Use_dl_runtime_resolve_opt (1 << 18)
+#define bit_Use_dl_runtime_resolve_slow (1 << 19)
/* CPUID Feature flags. */
@@ -101,6 +103,8 @@
# define index_I686 FEATURE_INDEX_1*FEATURE_SIZE
# define index_Prefer_MAP_32BIT_EXEC FEATURE_INDEX_1*FEATURE_SIZE
# define index_Prefer_No_VZEROUPPER FEATURE_INDEX_1*FEATURE_SIZE
+# define index_Use_dl_runtime_resolve_opt FEATURE_INDEX_1*FEATURE_SIZE
+# define index_Use_dl_runtime_resolve_slow FEATURE_INDEX_1*FEATURE_SIZE
# if defined (_LIBC) && !IS_IN (nonlib)
@@ -255,6 +259,8 @@ extern const struct cpu_features *__get_cpu_features (void)
# define index_I686 FEATURE_INDEX_1
# define index_Prefer_MAP_32BIT_EXEC FEATURE_INDEX_1
# define index_Prefer_No_VZEROUPPER FEATURE_INDEX_1
+# define index_Use_dl_runtime_resolve_opt FEATURE_INDEX_1
+# define index_Use_dl_runtime_resolve_slow FEATURE_INDEX_1
#endif /* !__ASSEMBLER__ */
diff --git a/sysdeps/x86_64/dl-machine.h b/sysdeps/x86_64/dl-machine.h
index 980ca73..e6a17e3 100644
--- a/sysdeps/x86_64/dl-machine.h
+++ b/sysdeps/x86_64/dl-machine.h
@@ -68,7 +68,10 @@ elf_machine_runtime_setup (struct link_map *l, int lazy, int profile)
Elf64_Addr *got;
extern void _dl_runtime_resolve_sse (ElfW(Word)) attribute_hidden;
extern void _dl_runtime_resolve_avx (ElfW(Word)) attribute_hidden;
+ extern void _dl_runtime_resolve_avx_slow (ElfW(Word)) attribute_hidden;
+ extern void _dl_runtime_resolve_avx_opt (ElfW(Word)) attribute_hidden;
extern void _dl_runtime_resolve_avx512 (ElfW(Word)) attribute_hidden;
+ extern void _dl_runtime_resolve_avx512_opt (ElfW(Word)) attribute_hidden;
extern void _dl_runtime_profile_sse (ElfW(Word)) attribute_hidden;
extern void _dl_runtime_profile_avx (ElfW(Word)) attribute_hidden;
extern void _dl_runtime_profile_avx512 (ElfW(Word)) attribute_hidden;
@@ -118,9 +121,26 @@ elf_machine_runtime_setup (struct link_map *l, int lazy, int profile)
indicated by the offset on the stack, and then jump to
the resolved address. */
if (HAS_ARCH_FEATURE (AVX512F_Usable))
- *(ElfW(Addr) *) (got + 2) = (ElfW(Addr)) &_dl_runtime_resolve_avx512;
+ {
+ if (HAS_ARCH_FEATURE (Use_dl_runtime_resolve_opt))
+ *(ElfW(Addr) *) (got + 2)
+ = (ElfW(Addr)) &_dl_runtime_resolve_avx512_opt;
+ else
+ *(ElfW(Addr) *) (got + 2)
+ = (ElfW(Addr)) &_dl_runtime_resolve_avx512;
+ }
else if (HAS_ARCH_FEATURE (AVX_Usable))
- *(ElfW(Addr) *) (got + 2) = (ElfW(Addr)) &_dl_runtime_resolve_avx;
+ {
+ if (HAS_ARCH_FEATURE (Use_dl_runtime_resolve_opt))
+ *(ElfW(Addr) *) (got + 2)
+ = (ElfW(Addr)) &_dl_runtime_resolve_avx_opt;
+ else if (HAS_ARCH_FEATURE (Use_dl_runtime_resolve_slow))
+ *(ElfW(Addr) *) (got + 2)
+ = (ElfW(Addr)) &_dl_runtime_resolve_avx_slow;
+ else
+ *(ElfW(Addr) *) (got + 2)
+ = (ElfW(Addr)) &_dl_runtime_resolve_avx;
+ }
else
*(ElfW(Addr) *) (got + 2) = (ElfW(Addr)) &_dl_runtime_resolve_sse;
}
diff --git a/sysdeps/x86_64/dl-trampoline.S b/sysdeps/x86_64/dl-trampoline.S
index 39b8771..3f812b8 100644
--- a/sysdeps/x86_64/dl-trampoline.S
+++ b/sysdeps/x86_64/dl-trampoline.S
@@ -18,6 +18,7 @@
#include <config.h>
#include <sysdep.h>
+#include <cpu-features.h>
#include <link-defines.h>
#ifndef DL_STACK_ALIGNMENT
@@ -87,9 +88,11 @@
# endif
# define VEC(i) zmm##i
# define _dl_runtime_resolve _dl_runtime_resolve_avx512
+# define _dl_runtime_resolve_opt _dl_runtime_resolve_avx512_opt
# define _dl_runtime_profile _dl_runtime_profile_avx512
# include "dl-trampoline.h"
# undef _dl_runtime_resolve
+# undef _dl_runtime_resolve_opt
# undef _dl_runtime_profile
# undef VEC
# undef VMOV
@@ -98,6 +101,8 @@
#else
strong_alias (_dl_runtime_resolve_avx, _dl_runtime_resolve_avx512)
.hidden _dl_runtime_resolve_avx512
+strong_alias (_dl_runtime_resolve_avx_opt, _dl_runtime_resolve_avx512_opt)
+ .hidden _dl_runtime_resolve_avx512_opt
strong_alias (_dl_runtime_profile_avx, _dl_runtime_profile_avx512)
.hidden _dl_runtime_profile_avx512
#endif
@@ -111,9 +116,11 @@ strong_alias (_dl_runtime_profile_avx, _dl_runtime_profile_avx512)
#endif
#define VEC(i) ymm##i
#define _dl_runtime_resolve _dl_runtime_resolve_avx
+#define _dl_runtime_resolve_opt _dl_runtime_resolve_avx_opt
#define _dl_runtime_profile _dl_runtime_profile_avx
#include "dl-trampoline.h"
#undef _dl_runtime_resolve
+#undef _dl_runtime_resolve_opt
#undef _dl_runtime_profile
#undef VEC
#undef VMOV
@@ -133,3 +140,18 @@ strong_alias (_dl_runtime_profile_avx, _dl_runtime_profile_avx512)
#define _dl_runtime_profile _dl_runtime_profile_sse
#undef RESTORE_AVX
#include "dl-trampoline.h"
+#undef _dl_runtime_resolve
+#undef _dl_runtime_profile
+#undef VMOV
+#undef VMOVA
+
+/* Used by _dl_runtime_resolve_avx_opt/_dl_runtime_resolve_avx512_opt
+ to preserve the full vector registers with zero upper bits. */
+#define VMOVA vmovdqa
+#if DL_RUNTIME_RESOLVE_REALIGN_STACK || VEC_SIZE <= DL_STACK_ALIGNMENT
+# define VMOV vmovdqa
+#else
+# define VMOV vmovdqu
+#endif
+#define _dl_runtime_resolve _dl_runtime_resolve_sse_vex
+#include "dl-trampoline.h"
diff --git a/sysdeps/x86_64/dl-trampoline.h b/sysdeps/x86_64/dl-trampoline.h
index b90836a..abe4471 100644
--- a/sysdeps/x86_64/dl-trampoline.h
+++ b/sysdeps/x86_64/dl-trampoline.h
@@ -50,6 +50,105 @@
#endif
.text
+#ifdef _dl_runtime_resolve_opt
+/* Use the smallest vector registers to preserve the full YMM/ZMM
+ registers to avoid SSE transition penalty. */
+
+# if VEC_SIZE == 32
+/* Check if the upper 128 bits in %ymm0 - %ymm7 registers are non-zero
+ and preserve %xmm0 - %xmm7 registers with the zero upper bits. Since
+ there is no SSE transition penalty on AVX512 processors which don't
+ support XGETBV with ECX == 1, _dl_runtime_resolve_avx512_slow isn't
+ provided. */
+ .globl _dl_runtime_resolve_avx_slow
+ .hidden _dl_runtime_resolve_avx_slow
+ .type _dl_runtime_resolve_avx_slow, @function
+ .align 16
+_dl_runtime_resolve_avx_slow:
+ cfi_startproc
+ cfi_adjust_cfa_offset(16) # Incorporate PLT
+ vorpd %ymm0, %ymm1, %ymm8
+ vorpd %ymm2, %ymm3, %ymm9
+ vorpd %ymm4, %ymm5, %ymm10
+ vorpd %ymm6, %ymm7, %ymm11
+ vorpd %ymm8, %ymm9, %ymm9
+ vorpd %ymm10, %ymm11, %ymm10
+ vpcmpeqd %xmm8, %xmm8, %xmm8
+ vorpd %ymm9, %ymm10, %ymm10
+ vptest %ymm10, %ymm8
+ # Preserve %ymm0 - %ymm7 registers if the upper 128 bits of any
+ # %ymm0 - %ymm7 registers aren't zero.
+ PRESERVE_BND_REGS_PREFIX
+ jnc _dl_runtime_resolve_avx
+ # Use vzeroupper to avoid SSE transition penalty.
+ vzeroupper
+ # Preserve %xmm0 - %xmm7 registers with the zero upper 128 bits
+ # when the upper 128 bits of %ymm0 - %ymm7 registers are zero.
+ PRESERVE_BND_REGS_PREFIX
+ jmp _dl_runtime_resolve_sse_vex
+ cfi_adjust_cfa_offset(-16) # Restore PLT adjustment
+ cfi_endproc
+ .size _dl_runtime_resolve_avx_slow, .-_dl_runtime_resolve_avx_slow
+# endif
+
+/* Use XGETBV with ECX == 1 to check which bits in vector registers are
+ non-zero and only preserve the non-zero lower bits with zero upper
+ bits. */
+ .globl _dl_runtime_resolve_opt
+ .hidden _dl_runtime_resolve_opt
+ .type _dl_runtime_resolve_opt, @function
+ .align 16
+_dl_runtime_resolve_opt:
+ cfi_startproc
+ cfi_adjust_cfa_offset(16) # Incorporate PLT
+ pushq %rax
+ cfi_adjust_cfa_offset(8)
+ cfi_rel_offset(%rax, 0)
+ pushq %rcx
+ cfi_adjust_cfa_offset(8)
+ cfi_rel_offset(%rcx, 0)
+ pushq %rdx
+ cfi_adjust_cfa_offset(8)
+ cfi_rel_offset(%rdx, 0)
+ movl $1, %ecx
+ xgetbv
+ movl %eax, %r11d
+ popq %rdx
+ cfi_adjust_cfa_offset(-8)
+ cfi_restore (%rdx)
+ popq %rcx
+ cfi_adjust_cfa_offset(-8)
+ cfi_restore (%rcx)
+ popq %rax
+ cfi_adjust_cfa_offset(-8)
+ cfi_restore (%rax)
+# if VEC_SIZE == 32
+ # For YMM registers, check if YMM state is in use.
+ andl $bit_YMM_state, %r11d
+ # Preserve %xmm0 - %xmm7 registers with the zero upper 128 bits if
+ # YMM state isn't in use.
+ PRESERVE_BND_REGS_PREFIX
+ jz _dl_runtime_resolve_sse_vex
+# elif VEC_SIZE == 64
+ # For ZMM registers, check if YMM state and ZMM state are in
+ # use.
+ andl $(bit_YMM_state | bit_ZMM0_15_state), %r11d
+ cmpl $bit_YMM_state, %r11d
+ # Preserve %xmm0 - %xmm7 registers with the zero upper 384 bits if
+ # neither YMM state nor ZMM state are in use.
+ PRESERVE_BND_REGS_PREFIX
+ jl _dl_runtime_resolve_sse_vex
+ # Preserve %ymm0 - %ymm7 registers with the zero upper 256 bits if
+ # ZMM state isn't in use.
+ PRESERVE_BND_REGS_PREFIX
+ je _dl_runtime_resolve_avx
+# else
+# error Unsupported VEC_SIZE!
+# endif
+ cfi_adjust_cfa_offset(-16) # Restore PLT adjustment
+ cfi_endproc
+ .size _dl_runtime_resolve_opt, .-_dl_runtime_resolve_opt
+#endif
.globl _dl_runtime_resolve
.hidden _dl_runtime_resolve
.type _dl_runtime_resolve, @function
@@ -162,7 +261,10 @@ _dl_runtime_resolve:
.size _dl_runtime_resolve, .-_dl_runtime_resolve
-#ifndef PROF
+/* To preserve %xmm0 - %xmm7 registers, dl-trampoline.h is included
+ twice, for _dl_runtime_resolve_sse and _dl_runtime_resolve_sse_vex.
+ But we don't need another _dl_runtime_profile for XMM registers. */
+#if !defined PROF && defined _dl_runtime_profile
# if (LR_VECTOR_OFFSET % VEC_SIZE) != 0
# error LR_VECTOR_OFFSET must be multples of VEC_SIZE
# endif
-----------------------------------------------------------------------
hooks/post-receive
--
GNU C Library master sources