This is the mail archive of the
libc-alpha@sourceware.org
mailing list for the glibc project.
[PATCH 09/15] Add elision to pthread_mutex_{try,timed,un,}lock
- From: Andi Kleen <andi at firstfloor dot org>
- To: libc-alpha at sourceware dot org
- Cc: hjl dot tools at gmail dot com,Andi Kleen <ak at linux dot intel dot com>
- Date: Sat, 22 Dec 2012 19:02:36 -0800
- Subject: [PATCH 09/15] Add elision to pthread_mutex_{try,timed,un,}lock
- References: <1356231762-23777-1-git-send-email-andi@firstfloor.org>
From: Andi Kleen <ak@linux.intel.com>
Add elision paths to the basic mutex locks.
The normal path has a check for RTM and upgrades the lock
to RTM when available.
We use a 4 byte value in the mutex to store the lock
elision adaptation state. This is separate from the adaptive
spin state and uses a separate field.
We use IFUNC to avoid any fast path overhead on systems without RTM.
The IFUNC implementation currently triggers some linker bugs,
that show up in the test suite when libc calls locks internally.
It works with HJ Lu's latest linker, but other linkers may
show additional failures in the test suite.
Linker known to work:
http://www.kernel.org/pub/linux/devel/binutils/binutils-2.23.51.0.8.tar.xz
Condition variables currently do not support elision.
Recursive mutexes and condition variables may be supported at some point,
but are not in the current implementation. Also "trylock" will
not automatically enable elision unless some other lock call
has been already called on the lock.
This adds a lot of files, but that's just for the various IFUNC
variants for multiarch.
The elision can be disabled with --disable-multi-arch
2012-12-22 Andi Kleen <ak@linux.intel.com>
Hongjiu Lu <hongjiu.lu@intel.com>
* nptl/pthread_mutex_lock.c (adaptive_lock): Add
(__pthread_mutex_lock): Add lock elision support.
* nptl/pthread_mutex_timedlock.c (pthread_mutex_timedlock): dito.
* nptl/pthread_mutex_trylock.c (__pthread_mutex_trylock): dito.
* nptl/pthread_mutex_unlock.c (__pthread_mutex_unlock_usercnt): dito.
* nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c: dito.
* nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h: dito.
* nptl/sysdeps/unix/sysv/linux/x86/multiarch/Makefile: Newfile
* nptl/sysdeps/unix/sysv/linux/x86/multiarch/force-elision.h: New file
* nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_cond_lock.c: dito.
* nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_lock-nortm.c: dito.
* nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_lock.c: dito.
* nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_timedlock-nortm.c: dito.
* nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_timedlock.c: dito.
* nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_trylock-nortm.c: dito.
* nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_trylock.c: dito.
* nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_unlock.c: dito.
---
nptl/pthread_mutex_lock.c | 108 ++++++++++++++-----
nptl/pthread_mutex_timedlock.c | 37 +++++++-
nptl/pthread_mutex_trylock.c | 37 ++++++-
nptl/pthread_mutex_unlock.c | 26 ++++-
.../unix/sysv/linux/pthread_mutex_cond_lock.c | 5 +
.../unix/sysv/linux/x86/bits/pthreadtypes.h | 13 ++-
.../sysdeps/unix/sysv/linux/x86/multiarch/Makefile | 5 +-
.../unix/sysv/linux/x86/multiarch/force-elision.h | 10 ++
.../linux/x86/multiarch/pthread_mutex_cond_lock.c | 5 +
.../linux/x86/multiarch/pthread_mutex_lock-nortm.c | 3 +
.../sysv/linux/x86/multiarch/pthread_mutex_lock.c | 32 ++++++
.../x86/multiarch/pthread_mutex_timedlock-nortm.c | 3 +
.../linux/x86/multiarch/pthread_mutex_timedlock.c | 29 +++++
.../x86/multiarch/pthread_mutex_trylock-nortm.c | 3 +
.../linux/x86/multiarch/pthread_mutex_trylock.c | 32 ++++++
.../linux/x86/multiarch/pthread_mutex_unlock.c | 2 +
16 files changed, 310 insertions(+), 40 deletions(-)
create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/multiarch/force-elision.h
create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_cond_lock.c
create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_lock-nortm.c
create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_lock.c
create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_timedlock-nortm.c
create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_timedlock.c
create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_trylock-nortm.c
create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_trylock.c
create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_unlock.c
diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c
index 3b1ef73..9785cd8 100644
--- a/nptl/pthread_mutex_lock.c
+++ b/nptl/pthread_mutex_lock.c
@@ -25,6 +25,10 @@
#include <lowlevellock.h>
#include <stap-probe.h>
+#ifndef lll_lock_elision
+#define lll_lock_elision(lock, try_lock, private) ({ \
+ lll_lock (lock, private); 0; })
+#endif
#ifndef LLL_MUTEX_LOCK
# define LLL_MUTEX_LOCK(mutex) \
@@ -34,12 +38,50 @@
# define LLL_ROBUST_MUTEX_LOCK(mutex, id) \
lll_robust_lock ((mutex)->__data.__lock, id, \
PTHREAD_ROBUST_MUTEX_PSHARED (mutex))
+# define LLL_MUTEX_LOCK_ELISION(mutex) \
+ lll_lock_elision ((mutex)->__data.__lock, (mutex)->__data.__spins, \
+ PTHREAD_MUTEX_PSHARED (mutex))
#endif
+#ifndef ENABLE_ELISION
+#define ENABLE_ELISION 0
+#endif
+
+#ifndef FORCE_ELISION
+#define FORCE_ELISION(m, s)
+#endif
static int __pthread_mutex_lock_full (pthread_mutex_t *mutex)
__attribute_noinline__;
+static inline __attribute__((always_inline)) void
+adaptive_lock (pthread_mutex_t *mutex)
+{
+ if (! __is_smp)
+ return;
+ if (LLL_MUTEX_TRYLOCK (mutex) != 0)
+ {
+ int cnt = 0;
+ int max_cnt = MIN (MAX_ADAPTIVE_COUNT, mutex->__data.__spins * 2 + 10);
+ do
+ {
+ if (cnt++ >= max_cnt)
+ {
+ LLL_MUTEX_LOCK (mutex);
+ break;
+ }
+
+#ifdef BUSY_WAIT_NOP
+ BUSY_WAIT_NOP;
+#endif
+ }
+ while (LLL_MUTEX_TRYLOCK (mutex) != 0);
+
+ mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
+ }
+ assert (mutex->__data.__owner == 0);
+}
+
int
__pthread_mutex_lock (mutex)
@@ -47,23 +89,36 @@ __pthread_mutex_lock (mutex)
{
assert (sizeof (mutex->__size) >= sizeof (mutex->__data));
- unsigned int type = PTHREAD_MUTEX_TYPE (mutex);
+ unsigned int type = PTHREAD_MUTEX_TYPE_EL (mutex);
LIBC_PROBE (mutex_entry, 1, mutex);
- if (__builtin_expect (type & ~PTHREAD_MUTEX_KIND_MASK_NP, 0))
+ if (__builtin_expect (type & ~(PTHREAD_MUTEX_KIND_MASK_NP
+ | PTHREAD_MUTEX_ELISION_FLAGS_NP), 0))
return __pthread_mutex_lock_full (mutex);
pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
- if (__builtin_expect (type, PTHREAD_MUTEX_TIMED_NP)
- == PTHREAD_MUTEX_TIMED_NP)
+ /* A switch would be likely faster */
+
+ if (__builtin_expect (type == PTHREAD_MUTEX_TIMED_NP, 1))
{
+ FORCE_ELISION (mutex, goto elision);
simple:
/* Normal mutex. */
LLL_MUTEX_LOCK (mutex);
assert (mutex->__data.__owner == 0);
}
+ else if (__builtin_expect (type == PTHREAD_MUTEX_TIMED_ELISION_NP, 1))
+ {
+ elision: __attribute__((unused))
+ if (!ENABLE_ELISION)
+ goto simple;
+ /* Don't record owner or users for elision case. */
+ int ret = LLL_MUTEX_LOCK_ELISION (mutex);
+ assert (mutex->__data.__owner == 0);
+ return ret;
+ }
else if (__builtin_expect (type == PTHREAD_MUTEX_RECURSIVE_NP, 1))
{
/* Recursive mutex. */
@@ -89,31 +144,26 @@ __pthread_mutex_lock (mutex)
}
else if (__builtin_expect (type == PTHREAD_MUTEX_ADAPTIVE_NP, 1))
{
- if (! __is_smp)
- goto simple;
-
- if (LLL_MUTEX_TRYLOCK (mutex) != 0)
- {
- int cnt = 0;
- int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
- mutex->__data.__spins * 2 + 10);
- do
- {
- if (cnt++ >= max_cnt)
- {
- LLL_MUTEX_LOCK (mutex);
- break;
- }
-
-#ifdef BUSY_WAIT_NOP
- BUSY_WAIT_NOP;
-#endif
- }
- while (LLL_MUTEX_TRYLOCK (mutex) != 0);
-
- mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
- }
- assert (mutex->__data.__owner == 0);
+ FORCE_ELISION (mutex, goto elision_adaptive);
+ adaptive:
+ adaptive_lock (mutex);
+ }
+ else if (type == PTHREAD_MUTEX_TIMED_ELISION_NP)
+ goto elision;
+ else if (type == PTHREAD_MUTEX_TIMED_NO_ELISION_NP)
+ goto simple;
+ else if (type == PTHREAD_MUTEX_ADAPTIVE_NO_ELISION_NP)
+ goto adaptive;
+ else if (type == PTHREAD_MUTEX_ADAPTIVE_ELISION_NP)
+ {
+ elision_adaptive: __attribute__((unused))
+ if (!ENABLE_ELISION)
+ goto adaptive;
+ if (!lll_trylock_elision (mutex->__data.__lock, mutex->__data.__elision))
+ return 0;
+ adaptive_lock (mutex);
+ /* No owner for elision */
+ return 0;
}
else
{
diff --git a/nptl/pthread_mutex_timedlock.c b/nptl/pthread_mutex_timedlock.c
index 52fa74c..ef358ec 100644
--- a/nptl/pthread_mutex_timedlock.c
+++ b/nptl/pthread_mutex_timedlock.c
@@ -25,6 +25,17 @@
#include <stap-probe.h>
+#ifndef lll_timedlock_elision
+#define lll_timedlock_elision(a,dummy,b,c) lll_timedlock(a, b, c)
+#endif
+
+#ifndef ENABLE_ELISION
+#define ENABLE_ELISION 0
+#endif
+
+#ifndef FORCE_ELISION
+#define FORCE_ELISION(m, s)
+#endif
int
pthread_mutex_timedlock (mutex, abstime)
@@ -40,7 +51,7 @@ pthread_mutex_timedlock (mutex, abstime)
/* We must not check ABSTIME here. If the thread does not block
abstime must not be checked for a valid value. */
- switch (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex),
+ switch (__builtin_expect (PTHREAD_MUTEX_TYPE_EL (mutex),
PTHREAD_MUTEX_TIMED_NP))
{
/* Recursive mutex. */
@@ -78,13 +89,37 @@ pthread_mutex_timedlock (mutex, abstime)
/* FALLTHROUGH */
case PTHREAD_MUTEX_TIMED_NP:
+ FORCE_ELISION (mutex, goto elision);
+ case PTHREAD_MUTEX_TIMED_NO_ELISION_NP:
simple:
/* Normal mutex. */
result = lll_timedlock (mutex->__data.__lock, abstime,
PTHREAD_MUTEX_PSHARED (mutex));
break;
+ case PTHREAD_MUTEX_TIMED_ELISION_NP:
+ elision: __attribute__((unused))
+ /* Don't record ownership */
+ if (!ENABLE_ELISION)
+ goto simple;
+ return lll_timedlock_elision (mutex->__data.__lock,
+ mutex->__data.__spins,
+ abstime,
+ PTHREAD_MUTEX_PSHARED (mutex));
+
+
+ case PTHREAD_MUTEX_ADAPTIVE_ELISION_NP:
+ adaptive_elision: __attribute__((unused))
+ if (ENABLE_ELISION
+ && !lll_trylock_elision (mutex->__data.__lock, mutex->__data.__elision))
+ return 0;
+ goto adaptive;
+
case PTHREAD_MUTEX_ADAPTIVE_NP:
+ FORCE_ELISION (mutex, goto adaptive_elision);
+
+ case PTHREAD_MUTEX_ADAPTIVE_NO_ELISION_NP:
+ adaptive:
if (! __is_smp)
goto simple;
diff --git a/nptl/pthread_mutex_trylock.c b/nptl/pthread_mutex_trylock.c
index 841b3d6..19aceed 100644
--- a/nptl/pthread_mutex_trylock.c
+++ b/nptl/pthread_mutex_trylock.c
@@ -22,6 +22,16 @@
#include "pthreadP.h"
#include <lowlevellock.h>
+#ifndef lll_trylock_elision
+#define lll_trylock_elision(a,t) lll_trylock(a)
+#endif
+
+#ifndef ENABLE_ELISION
+#define ENABLE_ELISION 0
+#endif
+
+/* We don't force elision in trylock, because this can lead to inconsistent
+ lock state if the lock was actually busy. */
int
__pthread_mutex_trylock (mutex)
@@ -30,7 +40,7 @@ __pthread_mutex_trylock (mutex)
int oldval;
pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
- switch (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex),
+ switch (__builtin_expect (PTHREAD_MUTEX_TYPE_EL (mutex),
PTHREAD_MUTEX_TIMED_NP))
{
/* Recursive mutex. */
@@ -57,12 +67,26 @@ __pthread_mutex_trylock (mutex)
}
break;
- case PTHREAD_MUTEX_ERRORCHECK_NP:
+ case PTHREAD_MUTEX_TIMED_ELISION_NP:
+ case PTHREAD_MUTEX_ADAPTIVE_ELISION_NP:
+ if (!ENABLE_ELISION)
+ goto normal;
+ if (lll_trylock_elision (mutex->__data.__lock,
+ mutex->__data.__elision) != 0)
+ break;
+ /* Don't record the ownership. */
+ return 0;
+
case PTHREAD_MUTEX_TIMED_NP:
case PTHREAD_MUTEX_ADAPTIVE_NP:
+ case PTHREAD_MUTEX_ADAPTIVE_NO_ELISION_NP:
+ /*FALL THROUGH*/
+ case PTHREAD_MUTEX_TIMED_NO_ELISION_NP:
+ case PTHREAD_MUTEX_ERRORCHECK_NP:
+ normal:
/* Normal mutex. */
- if (lll_trylock (mutex->__data.__lock) != 0)
- break;
+ if (lll_trylock (mutex->__data.__lock) != 0)
+ break;
/* Record the ownership. */
mutex->__data.__owner = id;
@@ -378,4 +402,9 @@ __pthread_mutex_trylock (mutex)
return EBUSY;
}
+
+#ifndef __pthread_mutex_trylock
+#ifndef pthread_mutex_trylock
strong_alias (__pthread_mutex_trylock, pthread_mutex_trylock)
+#endif
+#endif
diff --git a/nptl/pthread_mutex_unlock.c b/nptl/pthread_mutex_unlock.c
index 84e7415..725c43d 100644
--- a/nptl/pthread_mutex_unlock.c
+++ b/nptl/pthread_mutex_unlock.c
@@ -23,6 +23,14 @@
#include <lowlevellock.h>
#include <stap-probe.h>
+#ifndef lll_unlock_elision
+#define lll_unlock_elision(a,b) ({ lll_unlock (a,b); 0; })
+#endif
+
+#ifndef ENABLE_ELISION
+#define ENABLE_ELISION 0
+#endif
+
static int
internal_function
__pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)
@@ -34,8 +42,9 @@ __pthread_mutex_unlock_usercnt (mutex, decr)
pthread_mutex_t *mutex;
int decr;
{
- int type = PTHREAD_MUTEX_TYPE (mutex);
- if (__builtin_expect (type & ~PTHREAD_MUTEX_KIND_MASK_NP, 0))
+ int type = PTHREAD_MUTEX_TYPE_EL (mutex);
+ if (__builtin_expect (type &
+ ~(PTHREAD_MUTEX_KIND_MASK_NP|PTHREAD_MUTEX_ELISION_FLAGS_NP), 0))
return __pthread_mutex_unlock_full (mutex, decr);
if (__builtin_expect (type, PTHREAD_MUTEX_TIMED_NP)
@@ -55,6 +64,15 @@ __pthread_mutex_unlock_usercnt (mutex, decr)
return 0;
}
+ else if (__builtin_expect (type == PTHREAD_MUTEX_TIMED_ELISION_NP, 1)
+ || (type == PTHREAD_MUTEX_ADAPTIVE_ELISION_NP))
+ {
+ if (!ENABLE_ELISION)
+ goto normal;
+ /* Don't reset the owner/users fields for elision */
+ return lll_unlock_elision (mutex->__data.__lock,
+ PTHREAD_MUTEX_PSHARED (mutex));
+ }
else if (__builtin_expect (type == PTHREAD_MUTEX_RECURSIVE_NP, 1))
{
/* Recursive mutex. */
@@ -66,7 +84,9 @@ __pthread_mutex_unlock_usercnt (mutex, decr)
return 0;
goto normal;
}
- else if (__builtin_expect (type == PTHREAD_MUTEX_ADAPTIVE_NP, 1))
+ type &= ~PTHREAD_MUTEX_ELISION_FLAGS_NP;
+ if (__builtin_expect (type == PTHREAD_MUTEX_ADAPTIVE_NP, 1) ||
+ __builtin_expect (type == PTHREAD_MUTEX_TIMED_NP, 1))
goto normal;
else
{
diff --git a/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c
index b417da5..082196a 100644
--- a/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c
+++ b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c
@@ -2,6 +2,11 @@
#define LLL_MUTEX_LOCK(mutex) \
lll_cond_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
+
+/* Not actually elided so far. Needed? */
+#define LLL_MUTEX_LOCK_ELISION(mutex) \
+ ({ lll_cond_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex)); 0; })
+
#define LLL_MUTEX_TRYLOCK(mutex) \
lll_cond_trylock ((mutex)->__data.__lock)
#define LLL_ROBUST_MUTEX_LOCK(mutex, id) \
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h
index e22947f..10fab9b 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h
@@ -101,14 +101,23 @@ typedef union
binary compatibility. */
int __kind;
#ifdef __x86_64__
- int __spins;
+ short __spins;
+ short __elision;
__pthread_list_t __list;
# define __PTHREAD_MUTEX_HAVE_PREV 1
+# define __PTHREAD_MUTEX_HAVE_ELISION 1
#else
unsigned int __nusers;
__extension__ union
{
- int __spins;
+ struct
+ {
+ short __espins;
+ short __elision;
+# define __spins d.__espins
+# define __elision d.__elision
+# define __PTHREAD_MUTEX_HAVE_ELISION 2
+ } d;
__pthread_slist_t __list;
};
#endif
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/multiarch/Makefile b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/Makefile
index 61b7552..eb82cea 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86/multiarch/Makefile
+++ b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/Makefile
@@ -1,3 +1,6 @@
libpthread-sysdep_routines += init-arch
libpthread-sysdep_routines += elision-lock elision-unlock elision-timed \
- elision-trylock
+ elision-trylock \
+ pthread_mutex_lock-nortm \
+ pthread_mutex_trylock-nortm \
+ pthread_mutex_timedlock-nortm \
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/multiarch/force-elision.h b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/force-elision.h
new file mode 100644
index 0000000..98fca78
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/force-elision.h
@@ -0,0 +1,10 @@
+/* Automatically enable elision for existing user lock kinds */
+#define FORCE_ELISION(m, s) \
+ if (ENABLE_ELISION \
+ && (m->__data.__kind & PTHREAD_MUTEX_ELISION_FLAGS_NP) == 0 \
+ && __pthread_force_elision \
+ && __is_smp) \
+ { \
+ mutex->__data.__kind |= PTHREAD_MUTEX_ELISION_NP; \
+ s; \
+ }
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_cond_lock.c b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_cond_lock.c
new file mode 100644
index 0000000..7681f63
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_cond_lock.c
@@ -0,0 +1,5 @@
+/* The cond lock is not actually elided yet, but we still need to handle
+ already elided locks and have a working ENABLE_ELISION. */
+#include "elision-conf.h"
+#define ENABLE_ELISION (__elision_available != 0)
+#include "sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c"
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_lock-nortm.c b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_lock-nortm.c
new file mode 100644
index 0000000..d0b8ea4
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_lock-nortm.c
@@ -0,0 +1,3 @@
+#define ENABLE_ELISION 0
+#define __pthread_mutex_lock __pthread_mutex_lock_nortm
+#include "nptl/pthread_mutex_lock.c"
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_lock.c b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_lock.c
new file mode 100644
index 0000000..b2b71d5
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_lock.c
@@ -0,0 +1,32 @@
+/* Elided IFUNC version of pthread_mutex_lock.
+ Copyright (C) 2011, 2012 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+#include "elision-conf.h"
+#include "force-elision.h"
+#include "init-arch.h"
+
+#define ENABLE_ELISION 1
+#define __pthread_mutex_lock __pthread_mutex_lock_rtm
+#include "nptl/pthread_mutex_lock.c"
+#undef __pthread_mutex_lock
+
+libm_ifunc (__pthread_mutex_lock,
+ (HAS_RTM ? __pthread_mutex_lock_rtm
+ : __pthread_mutex_lock_nortm))
+
+strong_alias (__pthread_mutex_lock, pthread_mutex_lock)
+libc_ifunc_hidden_def (__pthread_mutex_lock)
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_timedlock-nortm.c b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_timedlock-nortm.c
new file mode 100644
index 0000000..f01c319
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_timedlock-nortm.c
@@ -0,0 +1,3 @@
+#define ENABLE_ELISION 0
+#define pthread_mutex_timedlock __pthread_mutex_timedlock_nortm
+#include "nptl/pthread_mutex_timedlock.c"
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_timedlock.c b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_timedlock.c
new file mode 100644
index 0000000..fc51775
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_timedlock.c
@@ -0,0 +1,29 @@
+/* Elided IFUNC version of pthread_mutex_timedlock.
+ Copyright (C) 2011, 2012 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+#include "elision-conf.h"
+#include "force-elision.h"
+#include "init-arch.h"
+
+#define ENABLE_ELISION 1
+#define pthread_mutex_timedlock __pthread_mutex_timedlock_rtm
+#include "nptl/pthread_mutex_timedlock.c"
+#undef pthread_mutex_timedlock
+
+libm_ifunc (pthread_mutex_timedlock,
+ (HAS_RTM ? __pthread_mutex_timedlock_rtm
+ : __pthread_mutex_timedlock_nortm))
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_trylock-nortm.c b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_trylock-nortm.c
new file mode 100644
index 0000000..ef7b8e4
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_trylock-nortm.c
@@ -0,0 +1,3 @@
+#define ENABLE_ELISION 0
+#define __pthread_mutex_trylock __pthread_mutex_trylock_nortm
+#include "nptl/pthread_mutex_trylock.c"
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_trylock.c b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_trylock.c
new file mode 100644
index 0000000..27efe6e
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_trylock.c
@@ -0,0 +1,32 @@
+/* Elided IFUNC version of pthread_mutex_trylock.
+ Copyright (C) 2011, 2012 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+#include "elision-conf.h"
+#include "force-elision.h"
+#include "init-arch.h"
+
+#define ENABLE_ELISION 1
+#define __pthread_mutex_trylock __pthread_mutex_trylock_rtm
+#include "nptl/pthread_mutex_trylock.c"
+#undef __pthread_mutex_trylock
+
+libm_ifunc (__pthread_mutex_trylock,
+ (HAS_RTM ? __pthread_mutex_trylock_rtm
+ : __pthread_mutex_trylock_nortm))
+
+strong_alias (__pthread_mutex_trylock, pthread_mutex_trylock)
+libc_ifunc_hidden_def (__pthread_mutex_trylock)
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_unlock.c b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_unlock.c
new file mode 100644
index 0000000..3b29b28
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_mutex_unlock.c
@@ -0,0 +1,2 @@
+#define ENABLE_ELISION 1
+#include "nptl/pthread_mutex_unlock.c"
--
1.7.7.6