This is the mail archive of the
glibc-cvs@sourceware.org
mailing list for the glibc project.
GNU C Library master sources branch hjl/avx2/master updated. glibc-2.25-396-g8c6a4bf
- From: hjl at sourceware dot org
- To: glibc-cvs at sourceware dot org
- Date: 2 Jun 2017 12:52:58 -0000
- Subject: GNU C Library master sources branch hjl/avx2/master updated. glibc-2.25-396-g8c6a4bf
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "GNU C Library master sources".
The branch, hjl/avx2/master has been updated
via 8c6a4bf036055891376110091d0ef8f13a5ce911 (commit)
via 023795d16d3e97fcabcbcb78a160e02b4397614e (commit)
via 597630559fd0bfe3c1ae9fc8f898c229c28b1d1e (commit)
via a4a7009311e5fcf88093915bfae326bade3e2823 (commit)
from 31106555b16efc27dcd20c1e8cde13a6679a1930 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
http://sourceware.org/git/gitweb.cgi?p=glibc.git;a=commitdiff;h=8c6a4bf036055891376110091d0ef8f13a5ce911
commit 8c6a4bf036055891376110091d0ef8f13a5ce911
Author: H.J. Lu <hjl.tools@gmail.com>
Date: Fri Jun 2 05:52:09 2017 -0700
memrchr.S: memrchr.c
diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
index 68fd4f4..f206469 100644
--- a/sysdeps/x86_64/multiarch/Makefile
+++ b/sysdeps/x86_64/multiarch/Makefile
@@ -7,6 +7,7 @@ ifeq ($(subdir),string)
sysdep_routines += strncat-c stpncpy-c strncpy-c strcmp-ssse3 \
strcmp-sse2-unaligned strncmp-ssse3 \
memchr-sse2 rawmemchr-sse2 memchr-avx2 rawmemchr-avx2 \
+ memrchr-sse2 memrchr-avx2 \
memcmp-avx2-movbe \
memcmp-sse4 memcpy-ssse3 \
memmove-ssse3 \
diff --git a/sysdeps/x86_64/multiarch/memrchr-sse2.S b/sysdeps/x86_64/multiarch/memrchr-sse2.S
new file mode 100644
index 0000000..f518819
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/memrchr-sse2.S
@@ -0,0 +1,26 @@
+/* memrchr optimized with SSE2.
+ Copyright (C) 2017 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#if IS_IN (libc)
+# define __memrchr __memrchr_sse2
+
+# undef weak_alias
+# define weak_alias(__memrchr, memrchr)
+#endif
+
+#include "../memrchr.S"
diff --git a/sysdeps/x86_64/multiarch/memrchr.S b/sysdeps/x86_64/multiarch/memrchr.c
similarity index 59%
rename from sysdeps/x86_64/multiarch/memrchr.S
rename to sysdeps/x86_64/multiarch/memrchr.c
index 20a9e76..26e777c 100644
--- a/sysdeps/x86_64/multiarch/memrchr.S
+++ b/sysdeps/x86_64/multiarch/memrchr.c
@@ -17,39 +17,15 @@
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
-#include <sysdep.h>
-#include <init-arch.h>
-
/* Define multiple versions only for the definition in libc. */
#if IS_IN (libc)
- .text
-ENTRY(__memrchr)
- .type __memrchr, @gnu_indirect_function
- LOAD_RTLD_GLOBAL_RO_RDX
- HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
- jnz 1f
- HAS_ARCH_FEATURE (AVX2_Usable)
- jz 1f
- HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
- jz 1f
- leaq __memrchr_avx2(%rip), %rax
- ret
+# define memrchr __redirect_memrchr
+# include <string.h>
+# undef memrchr
-1: leaq __memrchr_sse2(%rip), %rax
- ret
-END(__memrchr)
+# define SYMBOL_NAME memrchr
+# include "string-sse2-avx2.h"
-# undef ENTRY
-# define ENTRY(name) \
- .type __memrchr_sse2, @function; \
- .p2align 4; \
- .globl __memrchr_sse2; \
- .hidden __memrchr_sse2; \
- __memrchr_sse2: cfi_startproc; \
- CALL_MCOUNT
-# undef END
-# define END(name) \
- cfi_endproc; .size __memrchr_sse2, .-__memrchr_sse2
+libc_ifunc_redirected (__redirect_memrchr, __memrchr, IFUNC_SELECTOR ());
+weak_alias (__memrchr, memrchr)
#endif
-
-#include "../memrchr.S"
http://sourceware.org/git/gitweb.cgi?p=glibc.git;a=commitdiff;h=023795d16d3e97fcabcbcb78a160e02b4397614e
commit 023795d16d3e97fcabcbcb78a160e02b4397614e
Author: H.J. Lu <hjl.tools@gmail.com>
Date: Fri Jun 2 05:51:12 2017 -0700
memchr/rawmemchr/wmemchr: string-sse2-avx2.h
diff --git a/sysdeps/x86/init-arch.h b/sysdeps/x86/init-arch.h
index 6cbc16b..44664a5 100644
--- a/sysdeps/x86/init-arch.h
+++ b/sysdeps/x86/init-arch.h
@@ -21,6 +21,18 @@
# include <ldsodefs.h>
#endif
+#define PASTER1(x,y) x##_##y
+#define EVALUATOR1(x,y) PASTER1 (x,y)
+#define PASTER2(x,y) __##x##_##y
+#define EVALUATOR2(x,y) PASTER2 (x,y)
+
+/* Basically set '__redirect_<symbol>' to use as type definition,
+ '__<symbol>_<variant>' as the optimized implementation and
+ '<symbol>_ifunc_selector' as the IFUNC selector. */
+#define REDIRECT_NAME EVALUATOR1 (__redirect, SYMBOL_NAME)
+#define OPTIMIZE(name) EVALUATOR2 (SYMBOL_NAME, name)
+#define IFUNC_SELECTOR EVALUATOR1 (SYMBOL_NAME, ifunc_selector)
+
#ifndef __x86_64__
/* Due to the reordering and the other nifty extensions in i686, it is
not really good to use heavily i586 optimized code on an i686. It's
diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
index 3daae7c..68fd4f4 100644
--- a/sysdeps/x86_64/multiarch/Makefile
+++ b/sysdeps/x86_64/multiarch/Makefile
@@ -6,8 +6,7 @@ ifeq ($(subdir),string)
sysdep_routines += strncat-c stpncpy-c strncpy-c strcmp-ssse3 \
strcmp-sse2-unaligned strncmp-ssse3 \
- memchr-avx2 rawmemchr-avx2 \
- memrchr-avx2 \
+ memchr-sse2 rawmemchr-sse2 memchr-avx2 rawmemchr-avx2 \
memcmp-avx2-movbe \
memcmp-sse4 memcpy-ssse3 \
memmove-ssse3 \
@@ -37,7 +36,7 @@ endif
ifeq ($(subdir),wcsmbs)
sysdep_routines += wmemcmp-sse4 wmemcmp-ssse3 wmemcmp-c \
- wmemchr-avx2 \
+ wmemchr-sse2 wmemchr-avx2 \
wmemcmp-avx2-movbe \
wcscpy-ssse3 wcscpy-c \
wcschr-avx2 \
diff --git a/sysdeps/x86/init-arch.h b/sysdeps/x86_64/multiarch/memchr-sse2.S
similarity index 54%
copy from sysdeps/x86/init-arch.h
copy to sysdeps/x86_64/multiarch/memchr-sse2.S
index 6cbc16b..12c9c70 100644
--- a/sysdeps/x86/init-arch.h
+++ b/sysdeps/x86_64/multiarch/memchr-sse2.S
@@ -1,5 +1,6 @@
-/* This file is part of the GNU C Library.
- Copyright (C) 2008-2017 Free Software Foundation, Inc.
+/* memchr optimized with SSE2.
+ Copyright (C) 2017 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
@@ -15,21 +16,20 @@
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
-#ifdef __ASSEMBLER__
-# include <cpu-features.h>
-#else
-# include <ldsodefs.h>
-#endif
+#if IS_IN (libc)
+# define memchr __memchr_sse2
-#ifndef __x86_64__
-/* Due to the reordering and the other nifty extensions in i686, it is
- not really good to use heavily i586 optimized code on an i686. It's
- better to use i486 code if it isn't an i586. */
-# if MINIMUM_ISA == 686
-# define USE_I586 0
-# define USE_I686 1
-# else
-# define USE_I586 (HAS_ARCH_FEATURE (I586) && !HAS_ARCH_FEATURE (I686))
-# define USE_I686 HAS_ARCH_FEATURE (I686)
+# ifdef SHARED
+# undef libc_hidden_builtin_def
+/* It doesn't make sense to send libc-internal memchr calls through a PLT.
+ The speedup we get from using AVX2 instructions is likely eaten away
+ by the indirect call in the PLT. */
+# define libc_hidden_builtin_def(name) \
+ .globl __GI_memchr; __GI_memchr = __memchr_sse2
# endif
+
+# undef strong_alias
+# define strong_alias(memchr, __memchr)
#endif
+
+#include "../memchr.S"
diff --git a/sysdeps/x86_64/multiarch/memchr.S b/sysdeps/x86_64/multiarch/memchr.S
deleted file mode 100644
index dee3fd1..0000000
--- a/sysdeps/x86_64/multiarch/memchr.S
+++ /dev/null
@@ -1,64 +0,0 @@
-/* Multiple versions of memchr
- All versions must be listed in ifunc-impl-list.c.
- Copyright (C) 2017 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <http://www.gnu.org/licenses/>. */
-
-#include <sysdep.h>
-#include <init-arch.h>
-
-/* Define multiple versions only for the definition in libc. */
-#if IS_IN (libc)
- .text
-ENTRY(memchr)
- .type memchr, @gnu_indirect_function
- LOAD_RTLD_GLOBAL_RO_RDX
- HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
- jnz 1f
- HAS_ARCH_FEATURE (AVX2_Usable)
- jz 1f
- HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
- jz 1f
- leaq __memchr_avx2(%rip), %rax
- ret
-
-1: leaq __memchr_sse2(%rip), %rax
- ret
-END(memchr)
-
-# undef ENTRY
-# define ENTRY(name) \
- .type __memchr_sse2, @function; \
- .p2align 4; \
- .globl __memchr_sse2; \
- .hidden __memchr_sse2; \
- __memchr_sse2: cfi_startproc; \
- CALL_MCOUNT
-# undef END
-# define END(name) \
- cfi_endproc; .size __memchr_sse2, .-__memchr_sse2
-
-# ifdef SHARED
-# undef libc_hidden_builtin_def
-/* It doesn't make sense to send libc-internal memchr calls through a PLT.
- The speedup we get from using AVX2 instructions is likely eaten away
- by the indirect call in the PLT. */
-# define libc_hidden_builtin_def(name) \
- .globl __GI_memchr; __GI_memchr = __memchr_sse2
-# endif
-#endif
-
-#include "../memchr.S"
diff --git a/sysdeps/x86/init-arch.h b/sysdeps/x86_64/multiarch/memchr.c
similarity index 53%
copy from sysdeps/x86/init-arch.h
copy to sysdeps/x86_64/multiarch/memchr.c
index 6cbc16b..5e4ee63 100644
--- a/sysdeps/x86/init-arch.h
+++ b/sysdeps/x86_64/multiarch/memchr.c
@@ -1,5 +1,7 @@
-/* This file is part of the GNU C Library.
- Copyright (C) 2008-2017 Free Software Foundation, Inc.
+/* Multiple versions of memchr
+ All versions must be listed in ifunc-impl-list.c.
+ Copyright (C) 2017 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
@@ -15,21 +17,15 @@
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
-#ifdef __ASSEMBLER__
-# include <cpu-features.h>
-#else
-# include <ldsodefs.h>
-#endif
+/* Define multiple versions only for the definition in libc. */
+#if IS_IN (libc)
+# define memchr __redirect_memchr
+# include <string.h>
+# undef memchr
+
+# define SYMBOL_NAME memchr
+# include "string-sse2-avx2.h"
-#ifndef __x86_64__
-/* Due to the reordering and the other nifty extensions in i686, it is
- not really good to use heavily i586 optimized code on an i686. It's
- better to use i486 code if it isn't an i586. */
-# if MINIMUM_ISA == 686
-# define USE_I586 0
-# define USE_I686 1
-# else
-# define USE_I586 (HAS_ARCH_FEATURE (I586) && !HAS_ARCH_FEATURE (I686))
-# define USE_I686 HAS_ARCH_FEATURE (I686)
-# endif
+libc_ifunc_redirected (__redirect_memchr, memchr, IFUNC_SELECTOR ());
+strong_alias (memchr, __memchr)
#endif
diff --git a/sysdeps/x86_64/multiarch/rawmemchr.S b/sysdeps/x86_64/multiarch/rawmemchr-sse2.S
similarity index 62%
rename from sysdeps/x86_64/multiarch/rawmemchr.S
rename to sysdeps/x86_64/multiarch/rawmemchr-sse2.S
index b938f13..a44a7ff 100644
--- a/sysdeps/x86_64/multiarch/rawmemchr.S
+++ b/sysdeps/x86_64/multiarch/rawmemchr-sse2.S
@@ -1,5 +1,4 @@
-/* Multiple versions of rawmemchr
- All versions must be listed in ifunc-impl-list.c.
+/* rawmemchr optimized with SSE2.
Copyright (C) 2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
@@ -17,39 +16,9 @@
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
-#include <sysdep.h>
-#include <init-arch.h>
-
/* Define multiple versions only for the definition in libc. */
#if IS_IN (libc)
- .text
-ENTRY(__rawmemchr)
- .type __rawmemchr, @gnu_indirect_function
- LOAD_RTLD_GLOBAL_RO_RDX
- HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
- jnz 1f
- HAS_ARCH_FEATURE (AVX2_Usable)
- jz 1f
- HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
- jz 1f
- leaq __rawmemchr_avx2(%rip), %rax
- ret
-
-1: leaq __rawmemchr_sse2(%rip), %rax
- ret
-END(__rawmemchr)
-
-# undef ENTRY
-# define ENTRY(name) \
- .type __rawmemchr_sse2, @function; \
- .p2align 4; \
- .globl __rawmemchr_sse2; \
- .hidden __rawmemchr_sse2; \
- __rawmemchr_sse2: cfi_startproc; \
- CALL_MCOUNT
-# undef END
-# define END(name) \
- cfi_endproc; .size __rawmemchr_sse2, .-__rawmemchr_sse2
+# define __rawmemchr __rawmemchr_sse2
# ifdef SHARED
/* It doesn't make sense to send libc-internal rawmemchr calls through a
@@ -59,6 +28,9 @@ END(__rawmemchr)
# define libc_hidden_def(name) \
.globl __GI___rawmemchr; __GI___rawmemchr = __rawmemchr_sse2
# endif
+
+# undefine weak_alias
+# define weak_alias(__rawmemchr, rawmemchr)
#endif
#include "../rawmemchr.S"
diff --git a/sysdeps/x86/init-arch.h b/sysdeps/x86_64/multiarch/rawmemchr.c
similarity index 53%
copy from sysdeps/x86/init-arch.h
copy to sysdeps/x86_64/multiarch/rawmemchr.c
index 6cbc16b..d81e9a8 100644
--- a/sysdeps/x86/init-arch.h
+++ b/sysdeps/x86_64/multiarch/rawmemchr.c
@@ -1,5 +1,7 @@
-/* This file is part of the GNU C Library.
- Copyright (C) 2008-2017 Free Software Foundation, Inc.
+/* Multiple versions of rawmemchr
+ All versions must be listed in ifunc-impl-list.c.
+ Copyright (C) 2017 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
@@ -15,21 +17,18 @@
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
-#ifdef __ASSEMBLER__
-# include <cpu-features.h>
-#else
-# include <ldsodefs.h>
-#endif
+/* Define multiple versions only for the definition in libc. */
+#if IS_IN (libc)
+# define rawmemchr __redirect_rawmemchr
+# define __rawmemchr __redirect___rawmemchr
+# include <string.h>
+# undef rawmemchr
+# undef __rawmemchr
+
+# define SYMBOL_NAME rawmemchr
+# include "string-sse2-avx2.h"
-#ifndef __x86_64__
-/* Due to the reordering and the other nifty extensions in i686, it is
- not really good to use heavily i586 optimized code on an i686. It's
- better to use i486 code if it isn't an i586. */
-# if MINIMUM_ISA == 686
-# define USE_I586 0
-# define USE_I686 1
-# else
-# define USE_I586 (HAS_ARCH_FEATURE (I586) && !HAS_ARCH_FEATURE (I686))
-# define USE_I686 HAS_ARCH_FEATURE (I686)
-# endif
+libc_ifunc_redirected (__redirect_rawmemchr, __rawmemchr,
+ IFUNC_SELECTOR ());
+weak_alias (__rawmemchr, rawmemchr)
#endif
diff --git a/sysdeps/x86_64/multiarch/string-sse2-avx2.h b/sysdeps/x86_64/multiarch/string-sse2-avx2.h
new file mode 100644
index 0000000..9b26086
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/string-sse2-avx2.h
@@ -0,0 +1,37 @@
+/* Common definition for ifunc selections of string functions optimized
+ with SSE2 and AVX2.
+ All versions must be listed in ifunc-impl-list.c.
+ Copyright (C) 2017 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#include <init-arch.h>
+
+extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden;
+
+static inline void *
+IFUNC_SELECTOR (void)
+{
+ const struct cpu_features* cpu_features = __get_cpu_features ();
+
+ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)
+ && CPU_FEATURES_ARCH_P (cpu_features, AVX2_Usable)
+ && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
+ return OPTIMIZE (avx2);
+
+ return OPTIMIZE (sse2);
+}
diff --git a/sysdeps/x86_64/multiarch/wmemchr.S b/sysdeps/x86_64/multiarch/wmemchr-sse2.S
similarity index 64%
rename from sysdeps/x86_64/multiarch/wmemchr.S
rename to sysdeps/x86_64/multiarch/wmemchr-sse2.S
index b79df23..49cd86a 100644
--- a/sysdeps/x86_64/multiarch/wmemchr.S
+++ b/sysdeps/x86_64/multiarch/wmemchr-sse2.S
@@ -1,5 +1,4 @@
-/* Multiple versions of wmemchr
- All versions must be listed in ifunc-impl-list.c.
+/* wmemchr optimized with SSE2.
Copyright (C) 2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
@@ -17,39 +16,9 @@
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
-#include <sysdep.h>
-#include <init-arch.h>
-
/* Define multiple versions only for the definition in libc. */
#if IS_IN (libc)
- .text
-ENTRY(__wmemchr)
- .type __wmemchr, @gnu_indirect_function
- LOAD_RTLD_GLOBAL_RO_RDX
- HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
- jnz 1f
- HAS_ARCH_FEATURE (AVX2_Usable)
- jz 1f
- HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
- jz 1f
- leaq __wmemchr_avx2(%rip), %rax
- ret
-
-1: leaq __wmemchr_sse2(%rip), %rax
- ret
-END(__wmemchr)
-
-# undef ENTRY
-# define ENTRY(name) \
- .type __wmemchr_sse2, @function; \
- .p2align 4; \
- .globl __wmemchr_sse2; \
- .hidden __wmemchr_sse2; \
- __wmemchr_sse2: cfi_startproc; \
- CALL_MCOUNT
-# undef END
-# define END(name) \
- cfi_endproc; .size __wmemchr_sse2, .-__wmemchr_sse2
+# define wmemchr __wmemchr_sse2
# ifdef SHARED
/* It doesn't make sense to send libc-internal wmemchr calls through a PLT.
@@ -62,6 +31,9 @@ END(__wmemchr)
# define libc_hidden_weak(name) \
.weak __GI_wmemchr; __GI_wmemchr = __wmemchr_sse2
# endif
+
+# undef weak_alias
+# define weak_alias(__wmemchr, wmemchr)
#endif
#include "../wmemchr.S"
diff --git a/sysdeps/x86/init-arch.h b/sysdeps/x86_64/multiarch/wmemchr.c
similarity index 53%
copy from sysdeps/x86/init-arch.h
copy to sysdeps/x86_64/multiarch/wmemchr.c
index 6cbc16b..9ece9df 100644
--- a/sysdeps/x86/init-arch.h
+++ b/sysdeps/x86_64/multiarch/wmemchr.c
@@ -1,5 +1,7 @@
-/* This file is part of the GNU C Library.
- Copyright (C) 2008-2017 Free Software Foundation, Inc.
+/* Multiple versions of wmemchr
+ All versions must be listed in ifunc-impl-list.c.
+ Copyright (C) 2017 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
@@ -15,21 +17,17 @@
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
-#ifdef __ASSEMBLER__
-# include <cpu-features.h>
-#else
-# include <ldsodefs.h>
-#endif
+/* Define multiple versions only for the definition in libc. */
+#if IS_IN (libc)
+# define wmemchr __redirect_wmemchr
+# define __wmemchr __redirect___wmemchr
+# include <wchar.h>
+# undef wmemchr
+# undef __wmemchr
+
+# define SYMBOL_NAME wmemchr
+# include "string-sse2-avx2.h"
-#ifndef __x86_64__
-/* Due to the reordering and the other nifty extensions in i686, it is
- not really good to use heavily i586 optimized code on an i686. It's
- better to use i486 code if it isn't an i586. */
-# if MINIMUM_ISA == 686
-# define USE_I586 0
-# define USE_I686 1
-# else
-# define USE_I586 (HAS_ARCH_FEATURE (I586) && !HAS_ARCH_FEATURE (I686))
-# define USE_I686 HAS_ARCH_FEATURE (I686)
-# endif
+libc_ifunc_redirected (__redirect_wmemchr, __wmemchr, IFUNC_SELECTOR ());
+weak_alias (__wmemchr, wmemchr)
#endif
http://sourceware.org/git/gitweb.cgi?p=glibc.git;a=commitdiff;h=597630559fd0bfe3c1ae9fc8f898c229c28b1d1e
commit 597630559fd0bfe3c1ae9fc8f898c229c28b1d1e
Author: H.J. Lu <hjl.tools@gmail.com>
Date: Fri Jun 2 05:18:22 2017 -0700
strrchr/wcsrchr: string-sse2-avx2.h
diff --git a/sysdeps/x86_64/multiarch/strrchr.c b/sysdeps/x86_64/multiarch/strrchr.c
index c434090..488c4d8 100644
--- a/sysdeps/x86_64/multiarch/strrchr.c
+++ b/sysdeps/x86_64/multiarch/strrchr.c
@@ -23,7 +23,7 @@
# undef strrchr
# define SYMBOL_NAME strrchr
-# include "strrchr.h"
+# include "string-sse2-avx2.h"
extern __typeof (__redirect_strrchr) __libc_strrchr;
diff --git a/sysdeps/x86_64/multiarch/strrchr.h b/sysdeps/x86_64/multiarch/strrchr.h
deleted file mode 100644
index 9aa815b..0000000
--- a/sysdeps/x86_64/multiarch/strrchr.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/* Common definition for strrchr and wcsrchr ifunc selections.
- All versions must be listed in ifunc-impl-list.c.
- Copyright (C) 2017 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <http://www.gnu.org/licenses/>. */
-
-#include <init-arch.h>
-
-/* strrchr and wcsrchr share the same logic for ifunc selection. */
-
-extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden;
-extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden;
-
-static inline void *
-IFUNC_SELECTOR (void)
-{
- const struct cpu_features* cpu_features = __get_cpu_features ();
-
- if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)
- && CPU_FEATURES_ARCH_P (cpu_features, AVX2_Usable)
- && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
- return OPTIMIZE (avx2);
-
- return OPTIMIZE (sse2);
-}
diff --git a/sysdeps/x86_64/multiarch/wcsrchr.c b/sysdeps/x86_64/multiarch/wcsrchr.c
index ea29083..2d55f56 100644
--- a/sysdeps/x86_64/multiarch/wcsrchr.c
+++ b/sysdeps/x86_64/multiarch/wcsrchr.c
@@ -23,7 +23,7 @@
# undef wcsrchr
# define SYMBOL_NAME wcsrchr
-# include "strrchr.h"
+# include "string-sse2-avx2.h"
extern __typeof (__redirect_wcsrchr) __libc_wcsrchr;
http://sourceware.org/git/gitweb.cgi?p=glibc.git;a=commitdiff;h=a4a7009311e5fcf88093915bfae326bade3e2823
commit a4a7009311e5fcf88093915bfae326bade3e2823
Author: H.J. Lu <hjl.tools@gmail.com>
Date: Fri Jun 2 04:55:46 2017 -0700
strrchr/wcsrchr AVX2: c
diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
index 06568cb..3daae7c 100644
--- a/sysdeps/x86_64/multiarch/Makefile
+++ b/sysdeps/x86_64/multiarch/Makefile
@@ -15,7 +15,7 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c strcmp-ssse3 \
memmove-ssse3-back \
memmove-avx512-no-vzeroupper strcasecmp_l-ssse3 \
strchr-avx2 strchrnul-avx2 \
- strrchr-avx2 \
+ strrchr-sse2 strrchr-avx2 \
strlen-avx2 strnlen-avx2 \
strncase_l-ssse3 strcat-ssse3 strncat-ssse3\
strcpy-ssse3 strncpy-ssse3 stpcpy-ssse3 stpncpy-ssse3 \
@@ -41,6 +41,6 @@ sysdep_routines += wmemcmp-sse4 wmemcmp-ssse3 wmemcmp-c \
wmemcmp-avx2-movbe \
wcscpy-ssse3 wcscpy-c \
wcschr-avx2 \
- wcsrchr-avx2 \
+ wcsrchr-sse2 wcsrchr-avx2 \
wcslen-avx2 wcsnlen-avx2
endif
diff --git a/sysdeps/x86_64/multiarch/strrchr.S b/sysdeps/x86_64/multiarch/strrchr-sse2.S
similarity index 58%
rename from sysdeps/x86_64/multiarch/strrchr.S
rename to sysdeps/x86_64/multiarch/strrchr-sse2.S
index e5b49a9..bc60e8f 100644
--- a/sysdeps/x86_64/multiarch/strrchr.S
+++ b/sysdeps/x86_64/multiarch/strrchr-sse2.S
@@ -1,5 +1,5 @@
-/* Multiple versions of strrchr
- Copyright (C) 2009-2017 Free Software Foundation, Inc.
+/* strrchr optimized with SSE2.
+ Copyright (C) 2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
@@ -16,48 +16,18 @@
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
-#include <sysdep.h>
-#include <init-arch.h>
-
-
-/* Define multiple versions only for the definition in libc. */
#if IS_IN (libc)
- .text
-ENTRY(strrchr)
- .type strrchr, @gnu_indirect_function
- LOAD_RTLD_GLOBAL_RO_RDX
- HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
- jnz 1f
- HAS_ARCH_FEATURE (AVX2_Usable)
- jz 1f
- HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
- jz 1f
- leaq __strrchr_avx2(%rip), %rax
- ret
-1:
- leaq __strrchr_sse2(%rip), %rax
- ret
-END(strrchr)
-
+# define strrchr __strrchr_sse2
-
-# undef ENTRY
-# define ENTRY(name) \
- .type __strrchr_sse2, @function; \
- .align 16; \
- .globl __strrchr_sse2; \
- .hidden __strrchr_sse2; \
- __strrchr_sse2: cfi_startproc; \
- CALL_MCOUNT
-# undef END
-# define END(name) \
- cfi_endproc; .size __strrchr_sse2, .-__strrchr_sse2
# undef libc_hidden_builtin_def
/* It doesn't make sense to send libc-internal strrchr calls through a PLT.
The speedup we get from using SSE4.2 instruction is likely eaten away
by the indirect call in the PLT. */
# define libc_hidden_builtin_def(name) \
.globl __GI_strrchr; __GI_strrchr = __strrchr_sse2
+
+# undef weak_alias
+# define weak_alias(strrchr, rindex)
#endif
#include "../strrchr.S"
diff --git a/sysdeps/x86_64/multiarch/wcsrchr.S b/sysdeps/x86_64/multiarch/strrchr.c
similarity index 50%
copy from sysdeps/x86_64/multiarch/wcsrchr.S
copy to sysdeps/x86_64/multiarch/strrchr.c
index 59276cb..c434090 100644
--- a/sysdeps/x86_64/multiarch/wcsrchr.S
+++ b/sysdeps/x86_64/multiarch/strrchr.c
@@ -1,5 +1,4 @@
-/* Multiple versions of wcsrchr
- All versions must be listed in ifunc-impl-list.c.
+/* Multiple versions of strrchr.
Copyright (C) 2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
@@ -17,39 +16,17 @@
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
-#include <sysdep.h>
-#include <init-arch.h>
-
-/* Define multiple versions only for the definition in libc. */
+/* Define multiple versions only for the definition in libc. */
#if IS_IN (libc)
- .text
-ENTRY(wcsrchr)
- .type wcsrchr, @gnu_indirect_function
- LOAD_RTLD_GLOBAL_RO_RDX
- HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
- jnz 1f
- HAS_ARCH_FEATURE (AVX2_Usable)
- jz 1f
- HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
- jz 1f
- leaq __wcsrchr_avx2(%rip), %rax
- ret
+# define strrchr __redirect_strrchr
+# include <string.h>
+# undef strrchr
-1: leaq __wcsrchr_sse2(%rip), %rax
- ret
-END(wcsrchr)
+# define SYMBOL_NAME strrchr
+# include "strrchr.h"
-# undef ENTRY
-# define ENTRY(name) \
- .type __wcsrchr_sse2, @function; \
- .p2align 4; \
- .globl __wcsrchr_sse2; \
- .hidden __wcsrchr_sse2; \
- __wcsrchr_sse2: cfi_startproc; \
- CALL_MCOUNT
-# undef END
-# define END(name) \
- cfi_endproc; .size __wcsrchr_sse2, .-__wcsrchr_sse2
-#endif
+extern __typeof (__redirect_strrchr) __libc_strrchr;
-#include "../wcsrchr.S"
+libc_ifunc_redirected (__redirect_strrchr, strrchr, IFUNC_SELECTOR ());
+weak_alias (strrchr, rindex);
+#endif
diff --git a/sysdeps/x86_64/multiarch/wcsrchr.S b/sysdeps/x86_64/multiarch/strrchr.h
similarity index 53%
copy from sysdeps/x86_64/multiarch/wcsrchr.S
copy to sysdeps/x86_64/multiarch/strrchr.h
index 59276cb..9aa815b 100644
--- a/sysdeps/x86_64/multiarch/wcsrchr.S
+++ b/sysdeps/x86_64/multiarch/strrchr.h
@@ -1,4 +1,4 @@
-/* Multiple versions of wcsrchr
+/* Common definition for strrchr and wcsrchr ifunc selections.
All versions must be listed in ifunc-impl-list.c.
Copyright (C) 2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
@@ -17,39 +17,22 @@
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
-#include <sysdep.h>
#include <init-arch.h>
-/* Define multiple versions only for the definition in libc. */
-#if IS_IN (libc)
- .text
-ENTRY(wcsrchr)
- .type wcsrchr, @gnu_indirect_function
- LOAD_RTLD_GLOBAL_RO_RDX
- HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
- jnz 1f
- HAS_ARCH_FEATURE (AVX2_Usable)
- jz 1f
- HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
- jz 1f
- leaq __wcsrchr_avx2(%rip), %rax
- ret
-
-1: leaq __wcsrchr_sse2(%rip), %rax
- ret
-END(wcsrchr)
-
-# undef ENTRY
-# define ENTRY(name) \
- .type __wcsrchr_sse2, @function; \
- .p2align 4; \
- .globl __wcsrchr_sse2; \
- .hidden __wcsrchr_sse2; \
- __wcsrchr_sse2: cfi_startproc; \
- CALL_MCOUNT
-# undef END
-# define END(name) \
- cfi_endproc; .size __wcsrchr_sse2, .-__wcsrchr_sse2
-#endif
-
-#include "../wcsrchr.S"
+/* strrchr and wcsrchr share the same logic for ifunc selection. */
+
+extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden;
+
+static inline void *
+IFUNC_SELECTOR (void)
+{
+ const struct cpu_features* cpu_features = __get_cpu_features ();
+
+ if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)
+ && CPU_FEATURES_ARCH_P (cpu_features, AVX2_Usable)
+ && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
+ return OPTIMIZE (avx2);
+
+ return OPTIMIZE (sse2);
+}
diff --git a/sysdeps/x86_64/multiarch/wcsrchr.S b/sysdeps/x86_64/multiarch/wcsrchr-sse2.S
similarity index 51%
copy from sysdeps/x86_64/multiarch/wcsrchr.S
copy to sysdeps/x86_64/multiarch/wcsrchr-sse2.S
index 59276cb..967a7b1 100644
--- a/sysdeps/x86_64/multiarch/wcsrchr.S
+++ b/sysdeps/x86_64/multiarch/wcsrchr-sse2.S
@@ -1,5 +1,4 @@
-/* Multiple versions of wcsrchr
- All versions must be listed in ifunc-impl-list.c.
+/* wcsrchr optimized with SSE2.
Copyright (C) 2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
@@ -17,39 +16,15 @@
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
-#include <sysdep.h>
-#include <init-arch.h>
-
-/* Define multiple versions only for the definition in libc. */
#if IS_IN (libc)
- .text
-ENTRY(wcsrchr)
- .type wcsrchr, @gnu_indirect_function
- LOAD_RTLD_GLOBAL_RO_RDX
- HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
- jnz 1f
- HAS_ARCH_FEATURE (AVX2_Usable)
- jz 1f
- HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
- jz 1f
- leaq __wcsrchr_avx2(%rip), %rax
- ret
-
-1: leaq __wcsrchr_sse2(%rip), %rax
- ret
-END(wcsrchr)
-
-# undef ENTRY
-# define ENTRY(name) \
- .type __wcsrchr_sse2, @function; \
- .p2align 4; \
- .globl __wcsrchr_sse2; \
- .hidden __wcsrchr_sse2; \
- __wcsrchr_sse2: cfi_startproc; \
- CALL_MCOUNT
-# undef END
-# define END(name) \
- cfi_endproc; .size __wcsrchr_sse2, .-__wcsrchr_sse2
+# define wcsrchr __wcsrchr_sse2
+
+# undef libc_hidden_builtin_def
+/* It doesn't make sense to send libc-internal wcsrchr calls through a PLT.
+ The speedup we get from using SSE4.2 instruction is likely eaten away
+ by the indirect call in the PLT. */
+# define libc_hidden_builtin_def(name) \
+ .globl __GI_wcsrchr; __GI_wcsrchr = __wcsrchr_sse2
#endif
#include "../wcsrchr.S"
diff --git a/sysdeps/x86_64/multiarch/wcsrchr.S b/sysdeps/x86_64/multiarch/wcsrchr.c
similarity index 50%
rename from sysdeps/x86_64/multiarch/wcsrchr.S
rename to sysdeps/x86_64/multiarch/wcsrchr.c
index 59276cb..ea29083 100644
--- a/sysdeps/x86_64/multiarch/wcsrchr.S
+++ b/sysdeps/x86_64/multiarch/wcsrchr.c
@@ -1,5 +1,4 @@
-/* Multiple versions of wcsrchr
- All versions must be listed in ifunc-impl-list.c.
+/* Multiple versions of wcsrchr.
Copyright (C) 2017 Free Software Foundation, Inc.
This file is part of the GNU C Library.
@@ -17,39 +16,16 @@
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
-#include <sysdep.h>
-#include <init-arch.h>
-
-/* Define multiple versions only for the definition in libc. */
+/* Define multiple versions only for the definition in libc. */
#if IS_IN (libc)
- .text
-ENTRY(wcsrchr)
- .type wcsrchr, @gnu_indirect_function
- LOAD_RTLD_GLOBAL_RO_RDX
- HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
- jnz 1f
- HAS_ARCH_FEATURE (AVX2_Usable)
- jz 1f
- HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
- jz 1f
- leaq __wcsrchr_avx2(%rip), %rax
- ret
+# define wcsrchr __redirect_wcsrchr
+# include <wchar.h>
+# undef wcsrchr
-1: leaq __wcsrchr_sse2(%rip), %rax
- ret
-END(wcsrchr)
+# define SYMBOL_NAME wcsrchr
+# include "strrchr.h"
-# undef ENTRY
-# define ENTRY(name) \
- .type __wcsrchr_sse2, @function; \
- .p2align 4; \
- .globl __wcsrchr_sse2; \
- .hidden __wcsrchr_sse2; \
- __wcsrchr_sse2: cfi_startproc; \
- CALL_MCOUNT
-# undef END
-# define END(name) \
- cfi_endproc; .size __wcsrchr_sse2, .-__wcsrchr_sse2
-#endif
+extern __typeof (__redirect_wcsrchr) __libc_wcsrchr;
-#include "../wcsrchr.S"
+libc_ifunc_redirected (__redirect_wcsrchr, wcsrchr, IFUNC_SELECTOR ());
+#endif
-----------------------------------------------------------------------
Summary of changes:
sysdeps/x86/init-arch.h | 12 +++++
sysdeps/x86_64/multiarch/Makefile | 10 ++--
sysdeps/x86_64/multiarch/memchr-sse2.S | 35 ++++++++++++++
sysdeps/x86_64/multiarch/memchr.S | 64 -------------------------
sysdeps/x86_64/multiarch/memchr.c | 31 ++++++++++++
sysdeps/x86_64/multiarch/memrchr-sse2.S | 26 ++++++++++
sysdeps/x86_64/multiarch/memrchr.S | 55 ----------------------
sysdeps/x86_64/multiarch/memrchr.c | 31 ++++++++++++
sysdeps/x86_64/multiarch/rawmemchr-sse2.S | 36 ++++++++++++++
sysdeps/x86_64/multiarch/rawmemchr.S | 64 -------------------------
sysdeps/x86_64/multiarch/rawmemchr.c | 34 ++++++++++++++
sysdeps/x86_64/multiarch/string-sse2-avx2.h | 37 +++++++++++++++
sysdeps/x86_64/multiarch/strrchr-sse2.S | 33 +++++++++++++
sysdeps/x86_64/multiarch/strrchr.S | 63 -------------------------
sysdeps/x86_64/multiarch/strrchr.c | 32 +++++++++++++
sysdeps/x86_64/multiarch/wcsrchr-sse2.S | 30 ++++++++++++
sysdeps/x86_64/multiarch/wcsrchr.S | 55 ----------------------
sysdeps/x86_64/multiarch/wcsrchr.c | 31 ++++++++++++
sysdeps/x86_64/multiarch/wmemchr-sse2.S | 39 +++++++++++++++
sysdeps/x86_64/multiarch/wmemchr.S | 67 ---------------------------
sysdeps/x86_64/multiarch/wmemchr.c | 33 +++++++++++++
21 files changed, 445 insertions(+), 373 deletions(-)
create mode 100644 sysdeps/x86_64/multiarch/memchr-sse2.S
delete mode 100644 sysdeps/x86_64/multiarch/memchr.S
create mode 100644 sysdeps/x86_64/multiarch/memchr.c
create mode 100644 sysdeps/x86_64/multiarch/memrchr-sse2.S
delete mode 100644 sysdeps/x86_64/multiarch/memrchr.S
create mode 100644 sysdeps/x86_64/multiarch/memrchr.c
create mode 100644 sysdeps/x86_64/multiarch/rawmemchr-sse2.S
delete mode 100644 sysdeps/x86_64/multiarch/rawmemchr.S
create mode 100644 sysdeps/x86_64/multiarch/rawmemchr.c
create mode 100644 sysdeps/x86_64/multiarch/string-sse2-avx2.h
create mode 100644 sysdeps/x86_64/multiarch/strrchr-sse2.S
delete mode 100644 sysdeps/x86_64/multiarch/strrchr.S
create mode 100644 sysdeps/x86_64/multiarch/strrchr.c
create mode 100644 sysdeps/x86_64/multiarch/wcsrchr-sse2.S
delete mode 100644 sysdeps/x86_64/multiarch/wcsrchr.S
create mode 100644 sysdeps/x86_64/multiarch/wcsrchr.c
create mode 100644 sysdeps/x86_64/multiarch/wmemchr-sse2.S
delete mode 100644 sysdeps/x86_64/multiarch/wmemchr.S
create mode 100644 sysdeps/x86_64/multiarch/wmemchr.c
hooks/post-receive
--
GNU C Library master sources